Skip to content

DocToken.hpp

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
#pragma once

#include "doxide.hpp"

/**
 * Documentation comment token types. Closing delimiters must be one shift
 * left of their opening counterparts.
 *
 * @ingroup developer
 */
enum DocTokenType : int {
  NONE = 0,
  OPEN_AFTER = 1 << 1,   // opening sequence for doc comment after entity
  OPEN_BEFORE = 1 << 2,  // opening sequence for doc comment before entity
  CLOSE = 1 << 3,        // closing sequence for doc comment
  COMMAND = 1 << 4,
  PARA = 1 << 5,
  LINE = 1 << 6,
  SENTENCE = 1 << 7,
  WHITESPACE = 1 << 8,
  WORD = 1 << 9,
  STAR = 1 << 10,
  SLASH = 1 << 11,
  ANY = ~0
};

/**
 * Documentation comment token patterns. Order is important, as a match to an
 * earlier pattern precludes a match to a later.
 *
 * @ingroup developer
 */
static auto regexes = {
  std::make_pair(OPEN_AFTER, std::regex("(?:/\\*\\*|/\\*!|///|//!)<[ \\t]?", regex_flags)),
  std::make_pair(OPEN_BEFORE, std::regex("(?:/\\*\\*|/\\*!|///|//!)[ \\t]?", regex_flags)),
  std::make_pair(CLOSE, std::regex("\\*/", regex_flags)),
  std::make_pair(COMMAND, std::regex("[@\\\\](?:param(?:\\[(?:in|out|in,out)\\])?|\\w+|@|\\\\|/|f[\\$\\[\\]])", regex_flags)),

  /* the end of a paragraph is two new lines  */
  std::make_pair(PARA, std::regex("(?:[ \\t]*\\n(?:[ \\t]*\\*(?!/))?[ \\t]?){2}", regex_flags)),

  /* the end of a line is one new line, as long as there is not an end of
   * comment to come */
  std::make_pair(LINE, std::regex("[ \\t]*\\n(?:[ \\t]*\\*(?!/))?[ \\t]?", regex_flags)),

  std::make_pair(SENTENCE, std::regex("[.!?]", regex_flags)),
  std::make_pair(WHITESPACE, std::regex("\\s+", regex_flags)),
  std::make_pair(WORD, std::regex("[^\\s\\*/]+", regex_flags)),
  std::make_pair(STAR, std::regex("\\*", regex_flags)),
  std::make_pair(SLASH, std::regex("/", regex_flags))
};

/**
 * Token.
 * 
 * @ingroup developer
 * 
 * A token is only valid for the lifetime of the Tokenizer that produced it,
 * as it contains a reference to a substring of the source file.
 */
struct DocToken {
  /**
   * Constructor.
   * 
   * @param type Token type.
   * @param value Token value.
   */
  DocToken(const DocTokenType type = NONE,
      std::string_view value = std::string_view());

  /**
   * Get token as string.
   */
  std::string_view str() const;

  /**
   * Get substring of the token as a string.
   * 
   * @param pos Position of the first character.
   */
  std::string_view substr(size_t pos = 0) const;

  /**
   * Token type.
   */
  DocTokenType type;

  /**
   * Iterator to first character.
   */
  std::string_view value;
};