1. package chroma
    
  2. 
    
  3. //go:generate enumer -text -type TokenType
    
  4. 
    
  5. // TokenType is the type of token to highlight.
    
  6. //
    
  7. // It is also an Emitter, emitting a single token of itself
    
  8. type TokenType int
    
  9. 
    
  10. // Set of TokenTypes.
    
  11. //
    
  12. // Categories of types are grouped in ranges of 1000, while sub-categories are in ranges of 100. For
    
  13. // example, the literal category is in the range 3000-3999. The sub-category for literal strings is
    
  14. // in the range 3100-3199.
    
  15. 
    
  16. // Meta token types.
    
  17. const (
    
  18. 	// Default background style.
    
  19. 	Background TokenType = -1 - iota
    
  20. 	// PreWrapper style.
    
  21. 	PreWrapper
    
  22. 	// Line style.
    
  23. 	Line
    
  24. 	// Line numbers in output.
    
  25. 	LineNumbers
    
  26. 	// Line numbers in output when in table.
    
  27. 	LineNumbersTable
    
  28. 	// Line higlight style.
    
  29. 	LineHighlight
    
  30. 	// Line numbers table wrapper style.
    
  31. 	LineTable
    
  32. 	// Line numbers table TD wrapper style.
    
  33. 	LineTableTD
    
  34. 	// Line number links.
    
  35. 	LineLink
    
  36. 	// Code line wrapper style.
    
  37. 	CodeLine
    
  38. 	// Input that could not be tokenised.
    
  39. 	Error
    
  40. 	// Other is used by the Delegate lexer to indicate which tokens should be handled by the delegate.
    
  41. 	Other
    
  42. 	// No highlighting.
    
  43. 	None
    
  44. 	// Used as an EOF marker / nil token
    
  45. 	EOFType TokenType = 0
    
  46. )
    
  47. 
    
  48. // Keywords.
    
  49. const (
    
  50. 	Keyword TokenType = 1000 + iota
    
  51. 	KeywordConstant
    
  52. 	KeywordDeclaration
    
  53. 	KeywordNamespace
    
  54. 	KeywordPseudo
    
  55. 	KeywordReserved
    
  56. 	KeywordType
    
  57. )
    
  58. 
    
  59. // Names.
    
  60. const (
    
  61. 	Name TokenType = 2000 + iota
    
  62. 	NameAttribute
    
  63. 	NameBuiltin
    
  64. 	NameBuiltinPseudo
    
  65. 	NameClass
    
  66. 	NameConstant
    
  67. 	NameDecorator
    
  68. 	NameEntity
    
  69. 	NameException
    
  70. 	NameFunction
    
  71. 	NameFunctionMagic
    
  72. 	NameKeyword
    
  73. 	NameLabel
    
  74. 	NameNamespace
    
  75. 	NameOperator
    
  76. 	NameOther
    
  77. 	NamePseudo
    
  78. 	NameProperty
    
  79. 	NameTag
    
  80. 	NameVariable
    
  81. 	NameVariableAnonymous
    
  82. 	NameVariableClass
    
  83. 	NameVariableGlobal
    
  84. 	NameVariableInstance
    
  85. 	NameVariableMagic
    
  86. )
    
  87. 
    
  88. // Literals.
    
  89. const (
    
  90. 	Literal TokenType = 3000 + iota
    
  91. 	LiteralDate
    
  92. 	LiteralOther
    
  93. )
    
  94. 
    
  95. // Strings.
    
  96. const (
    
  97. 	LiteralString TokenType = 3100 + iota
    
  98. 	LiteralStringAffix
    
  99. 	LiteralStringAtom
    
  100. 	LiteralStringBacktick
    
  101. 	LiteralStringBoolean
    
  102. 	LiteralStringChar
    
  103. 	LiteralStringDelimiter
    
  104. 	LiteralStringDoc
    
  105. 	LiteralStringDouble
    
  106. 	LiteralStringEscape
    
  107. 	LiteralStringHeredoc
    
  108. 	LiteralStringInterpol
    
  109. 	LiteralStringName
    
  110. 	LiteralStringOther
    
  111. 	LiteralStringRegex
    
  112. 	LiteralStringSingle
    
  113. 	LiteralStringSymbol
    
  114. )
    
  115. 
    
  116. // Literals.
    
  117. const (
    
  118. 	LiteralNumber TokenType = 3200 + iota
    
  119. 	LiteralNumberBin
    
  120. 	LiteralNumberFloat
    
  121. 	LiteralNumberHex
    
  122. 	LiteralNumberInteger
    
  123. 	LiteralNumberIntegerLong
    
  124. 	LiteralNumberOct
    
  125. )
    
  126. 
    
  127. // Operators.
    
  128. const (
    
  129. 	Operator TokenType = 4000 + iota
    
  130. 	OperatorWord
    
  131. )
    
  132. 
    
  133. // Punctuation.
    
  134. const (
    
  135. 	Punctuation TokenType = 5000 + iota
    
  136. )
    
  137. 
    
  138. // Comments.
    
  139. const (
    
  140. 	Comment TokenType = 6000 + iota
    
  141. 	CommentHashbang
    
  142. 	CommentMultiline
    
  143. 	CommentSingle
    
  144. 	CommentSpecial
    
  145. )
    
  146. 
    
  147. // Preprocessor "comments".
    
  148. const (
    
  149. 	CommentPreproc TokenType = 6100 + iota
    
  150. 	CommentPreprocFile
    
  151. )
    
  152. 
    
  153. // Generic tokens.
    
  154. const (
    
  155. 	Generic TokenType = 7000 + iota
    
  156. 	GenericDeleted
    
  157. 	GenericEmph
    
  158. 	GenericError
    
  159. 	GenericHeading
    
  160. 	GenericInserted
    
  161. 	GenericOutput
    
  162. 	GenericPrompt
    
  163. 	GenericStrong
    
  164. 	GenericSubheading
    
  165. 	GenericTraceback
    
  166. 	GenericUnderline
    
  167. )
    
  168. 
    
  169. // Text.
    
  170. const (
    
  171. 	Text TokenType = 8000 + iota
    
  172. 	TextWhitespace
    
  173. 	TextSymbol
    
  174. 	TextPunctuation
    
  175. )
    
  176. 
    
  177. // Aliases.
    
  178. const (
    
  179. 	Whitespace = TextWhitespace
    
  180. 
    
  181. 	Date = LiteralDate
    
  182. 
    
  183. 	String          = LiteralString
    
  184. 	StringAffix     = LiteralStringAffix
    
  185. 	StringBacktick  = LiteralStringBacktick
    
  186. 	StringChar      = LiteralStringChar
    
  187. 	StringDelimiter = LiteralStringDelimiter
    
  188. 	StringDoc       = LiteralStringDoc
    
  189. 	StringDouble    = LiteralStringDouble
    
  190. 	StringEscape    = LiteralStringEscape
    
  191. 	StringHeredoc   = LiteralStringHeredoc
    
  192. 	StringInterpol  = LiteralStringInterpol
    
  193. 	StringOther     = LiteralStringOther
    
  194. 	StringRegex     = LiteralStringRegex
    
  195. 	StringSingle    = LiteralStringSingle
    
  196. 	StringSymbol    = LiteralStringSymbol
    
  197. 
    
  198. 	Number            = LiteralNumber
    
  199. 	NumberBin         = LiteralNumberBin
    
  200. 	NumberFloat       = LiteralNumberFloat
    
  201. 	NumberHex         = LiteralNumberHex
    
  202. 	NumberInteger     = LiteralNumberInteger
    
  203. 	NumberIntegerLong = LiteralNumberIntegerLong
    
  204. 	NumberOct         = LiteralNumberOct
    
  205. )
    
  206. 
    
  207. var (
    
  208. 	StandardTypes = map[TokenType]string{
    
  209. 		Background:       "bg",
    
  210. 		PreWrapper:       "chroma",
    
  211. 		Line:             "line",
    
  212. 		LineNumbers:      "ln",
    
  213. 		LineNumbersTable: "lnt",
    
  214. 		LineHighlight:    "hl",
    
  215. 		LineTable:        "lntable",
    
  216. 		LineTableTD:      "lntd",
    
  217. 		LineLink:         "lnlinks",
    
  218. 		CodeLine:         "cl",
    
  219. 		Text:             "",
    
  220. 		Whitespace:       "w",
    
  221. 		Error:            "err",
    
  222. 		Other:            "x",
    
  223. 		// I have no idea what this is used for...
    
  224. 		// Escape:     "esc",
    
  225. 
    
  226. 		Keyword:            "k",
    
  227. 		KeywordConstant:    "kc",
    
  228. 		KeywordDeclaration: "kd",
    
  229. 		KeywordNamespace:   "kn",
    
  230. 		KeywordPseudo:      "kp",
    
  231. 		KeywordReserved:    "kr",
    
  232. 		KeywordType:        "kt",
    
  233. 
    
  234. 		Name:                 "n",
    
  235. 		NameAttribute:        "na",
    
  236. 		NameBuiltin:          "nb",
    
  237. 		NameBuiltinPseudo:    "bp",
    
  238. 		NameClass:            "nc",
    
  239. 		NameConstant:         "no",
    
  240. 		NameDecorator:        "nd",
    
  241. 		NameEntity:           "ni",
    
  242. 		NameException:        "ne",
    
  243. 		NameFunction:         "nf",
    
  244. 		NameFunctionMagic:    "fm",
    
  245. 		NameProperty:         "py",
    
  246. 		NameLabel:            "nl",
    
  247. 		NameNamespace:        "nn",
    
  248. 		NameOther:            "nx",
    
  249. 		NameTag:              "nt",
    
  250. 		NameVariable:         "nv",
    
  251. 		NameVariableClass:    "vc",
    
  252. 		NameVariableGlobal:   "vg",
    
  253. 		NameVariableInstance: "vi",
    
  254. 		NameVariableMagic:    "vm",
    
  255. 
    
  256. 		Literal:     "l",
    
  257. 		LiteralDate: "ld",
    
  258. 
    
  259. 		String:          "s",
    
  260. 		StringAffix:     "sa",
    
  261. 		StringBacktick:  "sb",
    
  262. 		StringChar:      "sc",
    
  263. 		StringDelimiter: "dl",
    
  264. 		StringDoc:       "sd",
    
  265. 		StringDouble:    "s2",
    
  266. 		StringEscape:    "se",
    
  267. 		StringHeredoc:   "sh",
    
  268. 		StringInterpol:  "si",
    
  269. 		StringOther:     "sx",
    
  270. 		StringRegex:     "sr",
    
  271. 		StringSingle:    "s1",
    
  272. 		StringSymbol:    "ss",
    
  273. 
    
  274. 		Number:            "m",
    
  275. 		NumberBin:         "mb",
    
  276. 		NumberFloat:       "mf",
    
  277. 		NumberHex:         "mh",
    
  278. 		NumberInteger:     "mi",
    
  279. 		NumberIntegerLong: "il",
    
  280. 		NumberOct:         "mo",
    
  281. 
    
  282. 		Operator:     "o",
    
  283. 		OperatorWord: "ow",
    
  284. 
    
  285. 		Punctuation: "p",
    
  286. 
    
  287. 		Comment:            "c",
    
  288. 		CommentHashbang:    "ch",
    
  289. 		CommentMultiline:   "cm",
    
  290. 		CommentPreproc:     "cp",
    
  291. 		CommentPreprocFile: "cpf",
    
  292. 		CommentSingle:      "c1",
    
  293. 		CommentSpecial:     "cs",
    
  294. 
    
  295. 		Generic:           "g",
    
  296. 		GenericDeleted:    "gd",
    
  297. 		GenericEmph:       "ge",
    
  298. 		GenericError:      "gr",
    
  299. 		GenericHeading:    "gh",
    
  300. 		GenericInserted:   "gi",
    
  301. 		GenericOutput:     "go",
    
  302. 		GenericPrompt:     "gp",
    
  303. 		GenericStrong:     "gs",
    
  304. 		GenericSubheading: "gu",
    
  305. 		GenericTraceback:  "gt",
    
  306. 		GenericUnderline:  "gl",
    
  307. 	}
    
  308. )
    
  309. 
    
  310. func (t TokenType) Parent() TokenType {
    
  311. 	if t%100 != 0 {
    
  312. 		return t / 100 * 100
    
  313. 	}
    
  314. 	if t%1000 != 0 {
    
  315. 		return t / 1000 * 1000
    
  316. 	}
    
  317. 	return 0
    
  318. }
    
  319. 
    
  320. func (t TokenType) Category() TokenType {
    
  321. 	return t / 1000 * 1000
    
  322. }
    
  323. 
    
  324. func (t TokenType) SubCategory() TokenType {
    
  325. 	return t / 100 * 100
    
  326. }
    
  327. 
    
  328. func (t TokenType) InCategory(other TokenType) bool {
    
  329. 	return t/1000 == other/1000
    
  330. }
    
  331. 
    
  332. func (t TokenType) InSubCategory(other TokenType) bool {
    
  333. 	return t/100 == other/100
    
  334. }
    
  335. 
    
  336. func (t TokenType) Emit(groups []string, _ *LexerState) Iterator {
    
  337. 	return Literator(Token{Type: t, Value: groups[0]})
    
  338. }
    
  339. 
    
  340. func (t TokenType) EmitterKind() string { return "token" }