Skip to content

DocTokenizer.hpp

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#pragma once

#include "doxide.hpp"
#include "DocToken.hpp"

/**
 * Documentation comment tokenizer.
 * 
 * @ingroup developer
 */
class DocTokenizer {
public:
  /**
   * Constructor.
   * 
   * @param comment Comment to tokenize.
   */
  DocTokenizer(const std::string_view& source);

  /**
   * Get the next token.
   *
   * @return Next token.
   *
   * The token is only valid for the lifetime of the DocTokenizer, as it
   * contains a reference to a substring of the source file. If no tokens
   * remain (i.e. the end of the file is reached), a token with a type of
   * `NONE` is returned.
   */
  DocToken next();

  /**
   * Consume tokens until stopping criterion.
   * 
   * @param stop Bitmask giving the token types on which to stop and return.
   * 
   * @return Last token consumed.
   * 
   * Tokens are consumed until one is encountered with a type in @p stop,
   * which is then returned. If @p stop is `ANY` then the next token is
   * returned.
   */
  DocToken consume(const int stop = ANY);

private:
  /**
   * Iterator over source.
   */
  std::string_view::const_iterator iter;

  /**
   * End of source.
   */
  std::string_view::const_iterator end;
};