1. package lexers

  2. 

  3. import (

  4. 	"regexp"

  5. 	"strings"

  6. 	"unicode/utf8"

  7. 

  8. 	"github.com/dlclark/regexp2"

  9. 

  10. 	. "github.com/alecthomas/chroma/v2" // nolint

  11. )

  12. 

  13. // Raku lexer.

  14. var Raku Lexer = Register(MustNewLexer(

  15. 	&Config{

  16. 		Name:    "Raku",

  17. 		Aliases: []string{"perl6", "pl6", "raku"},

  18. 		Filenames: []string{

  19. 			"*.pl", "*.pm", "*.nqp", "*.p6", "*.6pl", "*.p6l", "*.pl6", "*.6pm",

  20. 			"*.p6m", "*.pm6", "*.t", "*.raku", "*.rakumod", "*.rakutest", "*.rakudoc",

  21. 		},

  22. 		MimeTypes: []string{

  23. 			"text/x-perl6", "application/x-perl6",

  24. 			"text/x-raku", "application/x-raku",

  25. 		},

  26. 		DotAll: true,

  27. 	},

  28. 	rakuRules,

  29. ))

  30. 

  31. func rakuRules() Rules {

  32. 	type RakuToken int

  33. 

  34. 	const (

  35. 		rakuQuote RakuToken = iota

  36. 		rakuNameAttribute

  37. 		rakuPod

  38. 		rakuPodFormatter

  39. 		rakuPodDeclaration

  40. 		rakuMultilineComment

  41. 		rakuMatchRegex

  42. 		rakuSubstitutionRegex

  43. 	)

  44. 

  45. 	const (

  46. 		colonPairOpeningBrackets = `(?:<<|<|«|\(|\[|\{)`

  47. 		colonPairClosingBrackets = `(?:>>|>|»|\)|\]|\})`

  48. 		colonPairPattern         = `(?<!:)(?<colon>:)(?<key>\w[\w'-]*)(?<opening_delimiters>` + colonPairOpeningBrackets + `)`

  49. 		colonPairLookahead       = `(?=(:['\w-]+` +

  50. 			colonPairOpeningBrackets + `.+?` + colonPairClosingBrackets + `)?`

  51. 		namePattern           = `(?:(?!` + colonPairPattern + `)(?:::|[\w':-]))+`

  52. 		variablePattern       = `[$@%&]+[.^:?=!~]?` + namePattern

  53. 		globalVariablePattern = `[$@%&]+\*` + namePattern

  54. 	)

  55. 

  56. 	keywords := []string{

  57. 		`BEGIN`, `CATCH`, `CHECK`, `CLOSE`, `CONTROL`, `DOC`, `END`, `ENTER`, `FIRST`, `INIT`,

  58. 		`KEEP`, `LAST`, `LEAVE`, `NEXT`, `POST`, `PRE`, `QUIT`, `UNDO`, `anon`, `augment`, `but`,

  59. 		`class`, `constant`, `default`, `does`, `else`, `elsif`, `enum`, `for`, `gather`, `given`,

  60. 		`grammar`, `has`, `if`, `import`, `is`, `of`, `let`, `loop`, `made`, `make`, `method`,

  61. 		`module`, `multi`, `my`, `need`, `orwith`, `our`, `proceed`, `proto`, `repeat`, `require`,

  62. 		`where`, `return`, `return-rw`, `returns`, `->`, `-->`, `role`, `state`, `sub`, `no`,

  63. 		`submethod`, `subset`, `succeed`, `supersede`, `try`, `unit`, `unless`, `until`,

  64. 		`use`, `when`, `while`, `with`, `without`, `export`, `native`, `repr`, `required`, `rw`,

  65. 		`symbol`, `default`, `cached`, `DEPRECATED`, `dynamic`, `hidden-from-backtrace`, `nodal`,

  66. 		`pure`, `raw`, `start`, `react`, `supply`, `whenever`, `also`, `rule`, `token`, `regex`,

  67. 		`dynamic-scope`, `built`, `temp`,

  68. 	}

  69. 

  70. 	keywordsPattern := Words(`(?<!['\w:-])`, `(?!['\w:-])`, keywords...)

  71. 

  72. 	wordOperators := []string{

  73. 		`X`, `Z`, `R`, `after`, `and`, `andthen`, `before`, `cmp`, `div`, `eq`, `eqv`, `extra`, `ge`,

  74. 		`gt`, `le`, `leg`, `lt`, `mod`, `ne`, `or`, `orelse`, `x`, `xor`, `xx`, `gcd`, `lcm`,

  75. 		`but`, `min`, `max`, `^fff`, `fff^`, `fff`, `^ff`, `ff^`, `ff`, `so`, `not`, `unicmp`,

  76. 		`TR`, `o`, `(&)`, `(.)`, `(|)`, `(+)`, `(-)`, `(^)`, `coll`, `(elem)`, `(==)`,

  77. 		`(cont)`, `(<)`, `(<=)`, `(>)`, `(>=)`, `minmax`, `notandthen`, `S`,

  78. 	}

  79. 

  80. 	wordOperatorsPattern := Words(`(?<=^|\b|\s)`, `(?=$|\b|\s)`, wordOperators...)

  81. 

  82. 	operators := []string{

  83. 		`++`, `--`, `-`, `**`, `!`, `+`, `~`, `?`, `+^`, `~^`, `?^`, `^`, `*`, `/`, `%`, `%%`, `+&`,

  84. 		`+<`, `+>`, `~&`, `~<`, `~>`, `?&`, `+|`, `+^`, `~|`, `~^`, `?`, `?|`, `?^`, `&`, `^`,

  85. 		`<=>`, `^…^`, `^…`, `…^`, `…`, `...`, `...^`, `^...`, `^...^`, `..`, `..^`, `^..`, `^..^`,

  86. 		`::=`, `:=`, `!=`, `==`, `<=`, `<`, `>=`, `>`, `~~`, `===`, `&&`, `||`, `|`, `^^`, `//`,

  87. 		`??`, `!!`, `^fff^`, `^ff^`, `<==`, `==>`, `<<==`, `==>>`, `=>`, `=`, `<<`, `«`, `>>`, `»`,

  88. 		`,`, `>>.`, `».`, `.&`, `.=`, `.^`, `.?`, `.+`, `.*`, `.`, `∘`, `∩`, `⊍`, `∪`, `⊎`, `∖`,

  89. 		`⊖`, `≠`, `≤`, `≥`, `=:=`, `=~=`, `≅`, `∈`, `∉`, `≡`, `≢`, `∋`, `∌`, `⊂`, `⊄`, `⊆`, `⊈`,

  90. 		`⊃`, `⊅`, `⊇`, `⊉`, `:`, `!!!`, `???`, `¯`, `×`, `÷`, `−`, `⁺`, `⁻`,

  91. 	}

  92. 

  93. 	operatorsPattern := Words(``, ``, operators...)

  94. 

  95. 	builtinTypes := []string{

  96. 		`False`, `True`, `Order`, `More`, `Less`, `Same`, `Any`, `Array`, `Associative`, `AST`,

  97. 		`atomicint`, `Attribute`, `Backtrace`, `Backtrace::Frame`, `Bag`, `Baggy`, `BagHash`,

  98. 		`Blob`, `Block`, `Bool`, `Buf`, `Callable`, `CallFrame`, `Cancellation`, `Capture`,

  99. 		`CArray`, `Channel`, `Code`, `compiler`, `Complex`, `ComplexStr`, `CompUnit`,

  100. 		`CompUnit::PrecompilationRepository`, `CompUnit::Repository`, `Empty`,

  101. 		`CompUnit::Repository::FileSystem`, `CompUnit::Repository::Installation`, `Cool`,

  102. 		`CurrentThreadScheduler`, `CX::Warn`, `CX::Take`, `CX::Succeed`, `CX::Return`, `CX::Redo`,

  103. 		`CX::Proceed`, `CX::Next`, `CX::Last`, `CX::Emit`, `CX::Done`, `Cursor`, `Date`, `Dateish`,

  104. 		`DateTime`, `Distribution`, `Distribution::Hash`, `Distribution::Locally`,

  105. 		`Distribution::Path`, `Distribution::Resource`, `Distro`, `Duration`, `Encoding`,

  106. 		`Encoding::GlobalLexerRegistry`, `Endian`, `Enumeration`, `Exception`, `Failure`, `FatRat`, `Grammar`,

  107. 		`Hash`, `HyperWhatever`, `Instant`, `Int`, `int`, `int16`, `int32`, `int64`, `int8`, `str`,

  108. 		`IntStr`, `IO`, `IO::ArgFiles`, `IO::CatHandle`, `IO::Handle`, `IO::Notification`,

  109. 		`IO::Notification::Change`, `IO::Path`, `IO::Path::Cygwin`, `IO::Path::Parts`,

  110. 		`IO::Path::QNX`, `IO::Path::Unix`, `IO::Path::Win32`, `IO::Pipe`, `IO::Socket`,

  111. 		`IO::Socket::Async`, `IO::Socket::Async::ListenSocket`, `IO::Socket::INET`, `IO::Spec`,

  112. 		`IO::Spec::Cygwin`, `IO::Spec::QNX`, `IO::Spec::Unix`, `IO::Spec::Win32`, `IO::Special`,

  113. 		`Iterable`, `Iterator`, `Junction`, `Kernel`, `Label`, `List`, `Lock`, `Lock::Async`,

  114. 		`Lock::ConditionVariable`, `long`, `longlong`, `Macro`, `Map`, `Match`,

  115. 		`Metamodel::AttributeContainer`, `Metamodel::C3MRO`, `Metamodel::ClassHOW`,

  116. 		`Metamodel::ConcreteRoleHOW`, `Metamodel::CurriedRoleHOW`, `Metamodel::DefiniteHOW`,

  117. 		`Metamodel::Documenting`, `Metamodel::EnumHOW`, `Metamodel::Finalization`,

  118. 		`Metamodel::MethodContainer`, `Metamodel::Mixins`, `Metamodel::MROBasedMethodDispatch`,

  119. 		`Metamodel::MultipleInheritance`, `Metamodel::Naming`, `Metamodel::Primitives`,

  120. 		`Metamodel::PrivateMethodContainer`, `Metamodel::RoleContainer`, `Metamodel::RolePunning`,

  121. 		`Metamodel::Stashing`, `Metamodel::Trusting`, `Metamodel::Versioning`, `Method`, `Mix`,

  122. 		`MixHash`, `Mixy`, `Mu`, `NFC`, `NFD`, `NFKC`, `NFKD`, `Nil`, `Num`, `num32`, `num64`,

  123. 		`Numeric`, `NumStr`, `ObjAt`, `Order`, `Pair`, `Parameter`, `Perl`, `Pod::Block`,

  124. 		`Pod::Block::Code`, `Pod::Block::Comment`, `Pod::Block::Declarator`, `Pod::Block::Named`,

  125. 		`Pod::Block::Para`, `Pod::Block::Table`, `Pod::Heading`, `Pod::Item`, `Pointer`,

  126. 		`Positional`, `PositionalBindFailover`, `Proc`, `Proc::Async`, `Promise`, `Proxy`,

  127. 		`PseudoStash`, `QuantHash`, `RaceSeq`, `Raku`, `Range`, `Rat`, `Rational`, `RatStr`,

  128. 		`Real`, `Regex`, `Routine`, `Routine::WrapHandle`, `Scalar`, `Scheduler`, `Semaphore`,

  129. 		`Seq`, `Sequence`, `Set`, `SetHash`, `Setty`, `Signature`, `size_t`, `Slip`, `Stash`,

  130. 		`Str`, `StrDistance`, `Stringy`, `Sub`, `Submethod`, `Supplier`, `Supplier::Preserving`,

  131. 		`Supply`, `Systemic`, `Tap`, `Telemetry`, `Telemetry::Instrument::Thread`,

  132. 		`Telemetry::Instrument::ThreadPool`, `Telemetry::Instrument::Usage`, `Telemetry::Period`,

  133. 		`Telemetry::Sampler`, `Thread`, `Test`, `ThreadPoolScheduler`, `UInt`, `uint16`, `uint32`,

  134. 		`uint64`, `uint8`, `Uni`, `utf8`, `ValueObjAt`, `Variable`, `Version`, `VM`, `Whatever`,

  135. 		`WhateverCode`, `WrapHandle`, `NativeCall`,

  136. 		// Pragmas

  137. 		`precompilation`, `experimental`, `worries`, `MONKEY-TYPING`, `MONKEY-SEE-NO-EVAL`,

  138. 		`MONKEY-GUTS`, `fatal`, `lib`, `isms`, `newline`, `nqp`, `soft`,

  139. 		`strict`, `trace`, `variables`,

  140. 	}

  141. 

  142. 	builtinTypesPattern := Words(`(?<!['\w:-])`, `(?::[_UD])?(?!['\w:-])`, builtinTypes...)

  143. 

  144. 	builtinRoutines := []string{

  145. 		`ACCEPTS`, `abs`, `abs2rel`, `absolute`, `accept`, `accepts_type`, `accessed`, `acos`,

  146. 		`acosec`, `acosech`, `acosh`, `acotan`, `acotanh`, `acquire`, `act`, `action`, `actions`,

  147. 		`add`, `add_attribute`, `add_enum_value`, `add_fallback`, `add_method`, `add_parent`,

  148. 		`add_private_method`, `add_role`, `add_stash`, `add_trustee`, `addendum`, `adverb`, `after`,

  149. 		`all`, `allocate`, `allof`, `allowed`, `alternative-names`, `annotations`, `antipair`,

  150. 		`antipairs`, `any`, `anyof`, `api`, `app_lifetime`, `append`, `arch`, `archetypes`,

  151. 		`archname`, `args`, `ARGS-TO-CAPTURE`, `arity`, `Array`, `asec`, `asech`, `asin`, `asinh`,

  152. 		`ASSIGN-KEY`, `ASSIGN-POS`, `assuming`, `ast`, `at`, `atan`, `atan2`, `atanh`, `AT-KEY`,

  153. 		`atomic-assign`, `atomic-dec-fetch`, `atomic-fetch`, `atomic-fetch-add`, `atomic-fetch-dec`,

  154. 		`atomic-fetch-inc`, `atomic-fetch-sub`, `atomic-inc-fetch`, `AT-POS`, `attributes`, `auth`,

  155. 		`await`, `backend`, `backtrace`, `Bag`, `bag`, `Baggy`, `BagHash`, `bail-out`, `base`,

  156. 		`basename`, `base-repeating`, `base_type`, `batch`, `BIND-KEY`, `BIND-POS`, `bind-stderr`,

  157. 		`bind-stdin`, `bind-stdout`, `bind-udp`, `bits`, `bless`, `block`, `Bool`, `bool-only`,

  158. 		`bounds`, `break`, `Bridge`, `broken`, `BUILD`, `TWEAK`, `build-date`, `bytes`, `cache`,

  159. 		`callframe`, `calling-package`, `CALL-ME`, `callsame`, `callwith`, `can`, `cancel`,

  160. 		`candidates`, `cando`, `can-ok`, `canonpath`, `caps`, `caption`, `Capture`, `capture`,

  161. 		`cas`, `catdir`, `categorize`, `categorize-list`, `catfile`, `catpath`, `cause`, `ceiling`,

  162. 		`cglobal`, `changed`, `Channel`, `channel`, `chars`, `chdir`, `child`, `child-name`,

  163. 		`child-typename`, `chmod`, `chomp`, `chop`, `chr`, `chrs`, `chunks`, `cis`, `classify`,

  164. 		`classify-list`, `cleanup`, `clone`, `close`, `closed`, `close-stdin`, `cmp-ok`, `code`,

  165. 		`codename`, `codes`, `coerce_type`, `coll`, `collate`, `column`, `comb`, `combinations`,

  166. 		`command`, `comment`, `compiler`, `Complex`, `compose`, `composalizer`, `compose_type`,

  167. 		`compose_values`, `composer`, `compute_mro`, `condition`, `config`, `configure_destroy`,

  168. 		`configure_type_checking`, `conj`, `connect`, `constraints`, `construct`, `contains`,

  169. 		`content`, `contents`, `copy`, `cos`, `cosec`, `cosech`, `cosh`, `cotan`, `cotanh`, `count`,

  170. 		`count-only`, `cpu-cores`, `cpu-usage`, `CREATE`, `create_type`, `cross`, `cue`, `curdir`,

  171. 		`curupdir`, `d`, `Date`, `DateTime`, `day`, `daycount`, `day-of-month`, `day-of-week`,

  172. 		`day-of-year`, `days-in-month`, `dd-mm-yyyy`, `declaration`, `decode`, `decoder`, `deepmap`,

  173. 		`default`, `defined`, `DEFINITE`, `definite`, `delayed`, `delete`, `delete-by-compiler`,

  174. 		`DELETE-KEY`, `DELETE-POS`, `denominator`, `desc`, `DESTROY`, `destroyers`, `devnull`,

  175. 		`diag`, `did-you-mean`, `die`, `dies-ok`, `dir`, `dirname`, `distribution`, `dir-sep`,

  176. 		`DISTROnames`, `do`, `does`, `does-ok`, `done`, `done-testing`, `duckmap`, `dynamic`, `e`,

  177. 		`eager`, `earlier`, `elems`, `emit`, `enclosing`, `encode`, `encoder`, `encoding`, `end`,

  178. 		`endian`, `ends-with`, `enum_from_value`, `enum_value_list`, `enum_values`, `enums`, `EOF`,

  179. 		`eof`, `EVAL`, `eval-dies-ok`, `EVALFILE`, `eval-lives-ok`, `event`, `exception`,

  180. 		`excludes-max`, `excludes-min`, `EXISTS-KEY`, `EXISTS-POS`, `exit`, `exitcode`, `exp`,

  181. 		`expected`, `explicitly-manage`, `expmod`, `export_callback`, `extension`, `f`, `fail`,

  182. 		`FALLBACK`, `fails-like`, `fc`, `feature`, `file`, `filename`, `files`, `find`,

  183. 		`find_method`, `find_method_qualified`, `finish`, `first`, `flat`, `first-date-in-month`,

  184. 		`flatmap`, `flip`, `floor`, `flunk`, `flush`, `flush_cache`, `fmt`, `format`, `formatter`,

  185. 		`free-memory`, `freeze`, `from`, `from-list`, `from-loop`, `from-posix`, `from-slurpy`,

  186. 		`full`, `full-barrier`, `GENERATE-USAGE`, `generate_mixin`, `get`, `get_value`, `getc`,

  187. 		`gist`, `got`, `grab`, `grabpairs`, `grep`, `handle`, `handled`, `handles`, `hardware`,

  188. 		`has_accessor`, `Hash`, `hash`, `head`, `headers`, `hh-mm-ss`, `hidden`, `hides`, `hostname`,

  189. 		`hour`, `how`, `hyper`, `id`, `illegal`, `im`, `in`, `in-timezone`, `indent`, `index`,

  190. 		`indices`, `indir`, `infinite`, `infix`, `postcirumfix`, `cicumfix`, `install`,

  191. 		`install_method_cache`, `Instant`, `instead`, `Int`, `int-bounds`, `interval`, `in-timezone`,

  192. 		`invalid-str`, `invert`, `invocant`, `IO`, `IO::Notification.watch-path`, `is_trusted`,

  193. 		`is_type`, `isa`, `is-absolute`, `isa-ok`, `is-approx`, `is-deeply`, `is-hidden`,

  194. 		`is-initial-thread`, `is-int`, `is-lazy`, `is-leap-year`, `isNaN`, `isnt`, `is-prime`,

  195. 		`is-relative`, `is-routine`, `is-setting`, `is-win`, `item`, `iterator`, `join`, `keep`,

  196. 		`kept`, `KERNELnames`, `key`, `keyof`, `keys`, `kill`, `kv`, `kxxv`, `l`, `lang`, `last`,

  197. 		`lastcall`, `later`, `lazy`, `lc`, `leading`, `level`, `like`, `line`, `lines`, `link`,

  198. 		`List`, `list`, `listen`, `live`, `lives-ok`, `load`, `load-repo-id`, `load-unit`, `loaded`,

  199. 		`loads`, `local`, `lock`, `log`, `log10`, `lookup`, `lsb`, `made`, `MAIN`, `make`, `Map`,

  200. 		`map`, `match`, `max`, `maxpairs`, `merge`, `message`, `method`, `meta`, `method_table`,

  201. 		`methods`, `migrate`, `min`, `minmax`, `minpairs`, `minute`, `misplaced`, `Mix`, `mix`,

  202. 		`MixHash`, `mixin`, `mixin_attribute`, `Mixy`, `mkdir`, `mode`, `modified`, `month`, `move`,

  203. 		`mro`, `msb`, `multi`, `multiness`, `name`, `named`, `named_names`, `narrow`,

  204. 		`nativecast`, `native-descriptor`, `nativesizeof`, `need`, `new`, `new_type`,

  205. 		`new-from-daycount`, `new-from-pairs`, `next`, `nextcallee`, `next-handle`, `nextsame`,

  206. 		`nextwith`, `next-interesting-index`, `NFC`, `NFD`, `NFKC`, `NFKD`, `nice`, `nl-in`,

  207. 		`nl-out`, `nodemap`, `nok`, `normalize`, `none`, `norm`, `not`, `note`, `now`, `nude`,

  208. 		`Num`, `numerator`, `Numeric`, `of`, `offset`, `offset-in-hours`, `offset-in-minutes`,

  209. 		`ok`, `old`, `on-close`, `one`, `on-switch`, `open`, `opened`, `operation`, `optional`,

  210. 		`ord`, `ords`, `orig`, `os-error`, `osname`, `out-buffer`, `pack`, `package`, `package-kind`,

  211. 		`package-name`, `packages`, `Pair`, `pair`, `pairs`, `pairup`, `parameter`, `params`,

  212. 		`parent`, `parent-name`, `parents`, `parse`, `parse-base`, `parsefile`, `parse-names`,

  213. 		`parts`, `pass`, `path`, `path-sep`, `payload`, `peer-host`, `peer-port`, `periods`, `perl`,

  214. 		`permutations`, `phaser`, `pick`, `pickpairs`, `pid`, `placeholder`, `plan`, `plus`,

  215. 		`polar`, `poll`, `polymod`, `pop`, `pos`, `positional`, `posix`, `postfix`, `postmatch`,

  216. 		`precomp-ext`, `precomp-target`, `precompiled`, `pred`, `prefix`, `prematch`, `prepend`,

  217. 		`primary`, `print`, `printf`, `print-nl`, `print-to`, `private`, `private_method_names`,

  218. 		`private_method_table`, `proc`, `produce`, `Promise`, `promise`, `prompt`, `protect`,

  219. 		`protect-or-queue-on-recursion`, `publish_method_cache`, `pull-one`, `push`, `push-all`,

  220. 		`push-at-least`, `push-exactly`, `push-until-lazy`, `put`, `qualifier-type`, `quaternary`,

  221. 		`quit`, `r`, `race`, `radix`, `raku`, `rand`, `Range`, `range`, `Rat`, `raw`, `re`, `read`,

  222. 		`read-bits`, `read-int128`, `read-int16`, `read-int32`, `read-int64`, `read-int8`,

  223. 		`read-num32`, `read-num64`, `read-ubits`, `read-uint128`, `read-uint16`, `read-uint32`,

  224. 		`read-uint64`, `read-uint8`, `readchars`, `readonly`, `ready`, `Real`, `reallocate`,

  225. 		`reals`, `reason`, `rebless`, `receive`, `recv`, `redispatcher`, `redo`, `reduce`,

  226. 		`rel2abs`, `relative`, `release`, `remove`, `rename`, `repeated`, `replacement`,

  227. 		`replace-with`, `repo`, `repo-id`, `report`, `required`, `reserved`, `resolve`, `restore`,

  228. 		`result`, `resume`, `rethrow`, `return`, `return-rw`, `returns`, `reverse`, `right`,

  229. 		`rindex`, `rmdir`, `role`, `roles_to_compose`, `rolish`, `roll`, `rootdir`, `roots`,

  230. 		`rotate`, `rotor`, `round`, `roundrobin`, `routine-type`, `run`, `RUN-MAIN`, `rw`, `rwx`,

  231. 		`samecase`, `samemark`, `samewith`, `say`, `schedule-on`, `scheduler`, `scope`, `sec`,

  232. 		`sech`, `second`, `secondary`, `seek`, `self`, `send`, `Seq`, `Set`, `set`, `serial`,

  233. 		`set_hidden`, `set_name`, `set_package`, `set_rw`, `set_value`, `set_api`, `set_auth`,

  234. 		`set_composalizer`, `set_export_callback`, `set_is_mixin`, `set_mixin_attribute`,

  235. 		`set_package`, `set_ver`, `set_why`, `SetHash`, `Setty`, `set-instruments`,

  236. 		`setup_finalization`, `setup_mixin_cache`, `shape`, `share`, `shell`, `short-id`,

  237. 		`short-name`, `shortname`, `shift`, `sibling`, `sigil`, `sign`, `signal`, `signals`,

  238. 		`signature`, `sin`, `sinh`, `sink`, `sink-all`, `skip`, `skip-at-least`,

  239. 		`skip-at-least-pull-one`, `skip-one`, `skip-rest`, `sleep`, `sleep-timer`, `sleep-until`,

  240. 		`Slip`, `slip`, `slurp`, `slurp-rest`, `slurpy`, `snap`, `snapper`, `so`, `socket-host`,

  241. 		`socket-port`, `sort`, `source`, `source-package`, `spawn`, `SPEC`, `splice`, `split`,

  242. 		`splitdir`, `splitpath`, `sprintf`, `spurt`, `sqrt`, `squish`, `srand`, `stable`, `start`,

  243. 		`started`, `starts-with`, `status`, `stderr`, `stdout`, `STORE`, `store-file`,

  244. 		`store-repo-id`, `store-unit`, `Str`, `Stringy`, `sub_signature`, `subbuf`, `subbuf-rw`,

  245. 		`subname`, `subparse`, `subst`, `subst-mutate`, `substr`, `substr-eq`, `substr-rw`,

  246. 		`subtest`, `succ`, `sum`, `suffix`, `summary`, `Supply`, `symlink`, `T`, `t`, `tail`,

  247. 		`take`, `take-rw`, `tan`, `tanh`, `tap`, `target`, `target-name`, `tc`, `tclc`, `tell`,

  248. 		`term`, `tertiary`, `then`, `throttle`, `throw`, `throws-like`, `time`, `timezone`,

  249. 		`tmpdir`, `to`, `today`, `todo`, `toggle`, `to-posix`, `total`, `total-memory`, `trailing`,

  250. 		`trans`, `tree`, `trim`, `trim-leading`, `trim-trailing`, `truncate`, `truncated-to`,

  251. 		`trusts`, `try_acquire`, `trying`, `twigil`, `type`, `type_captures`, `type_check`,

  252. 		`typename`, `uc`, `udp`, `uncaught_handler`, `undefine`, `unimatch`, `unicmp`, `uniname`,

  253. 		`uninames`, `uninstall`, `uniparse`, `uniprop`, `uniprops`, `unique`, `unival`, `univals`,

  254. 		`unlike`, `unlink`, `unlock`, `unpack`, `unpolar`, `unset`, `unshift`, `unwrap`, `updir`,

  255. 		`USAGE`, `usage-name`, `use-ok`, `utc`, `val`, `value`, `values`, `VAR`, `variable`, `ver`,

  256. 		`verbose-config`, `Version`, `version`, `VMnames`, `volume`, `vow`, `w`, `wait`, `warn`,

  257. 		`watch`, `watch-path`, `week`, `weekday-of-month`, `week-number`, `week-year`, `WHAT`,

  258. 		`what`, `when`, `WHERE`, `WHEREFORE`, `WHICH`, `WHO`, `whole-second`, `WHY`, `why`,

  259. 		`with-lock-hidden-from-recursion-check`, `wordcase`, `words`, `workaround`, `wrap`,

  260. 		`write`, `write-bits`, `write-int128`, `write-int16`, `write-int32`, `write-int64`,

  261. 		`write-int8`, `write-num32`, `write-num64`, `write-ubits`, `write-uint128`, `write-uint16`,

  262. 		`write-uint32`, `write-uint64`, `write-uint8`, `write-to`, `x`, `yada`, `year`, `yield`,

  263. 		`yyyy-mm-dd`, `z`, `zip`, `zip-latest`, `HOW`, `s`, `DEPRECATED`, `trait_mod`,

  264. 	}

  265. 

  266. 	builtinRoutinesPattern := Words(`(?<!['\w:-])`, `(?!['\w-])`, builtinRoutines...)

  267. 

  268. 	// A map of opening and closing brackets

  269. 	brackets := map[rune]rune{

  270. 		'\u0028': '\u0029', '\u003c': '\u003e', '\u005b': '\u005d',

  271. 		'\u007b': '\u007d', '\u00ab': '\u00bb', '\u0f3a': '\u0f3b',

  272. 		'\u0f3c': '\u0f3d', '\u169b': '\u169c', '\u2018': '\u2019',

  273. 		'\u201a': '\u2019', '\u201b': '\u2019', '\u201c': '\u201d',

  274. 		'\u201e': '\u201d', '\u201f': '\u201d', '\u2039': '\u203a',

  275. 		'\u2045': '\u2046', '\u207d': '\u207e', '\u208d': '\u208e',

  276. 		'\u2208': '\u220b', '\u2209': '\u220c', '\u220a': '\u220d',

  277. 		'\u2215': '\u29f5', '\u223c': '\u223d', '\u2243': '\u22cd',

  278. 		'\u2252': '\u2253', '\u2254': '\u2255', '\u2264': '\u2265',

  279. 		'\u2266': '\u2267', '\u2268': '\u2269', '\u226a': '\u226b',

  280. 		'\u226e': '\u226f', '\u2270': '\u2271', '\u2272': '\u2273',

  281. 		'\u2274': '\u2275', '\u2276': '\u2277', '\u2278': '\u2279',

  282. 		'\u227a': '\u227b', '\u227c': '\u227d', '\u227e': '\u227f',

  283. 		'\u2280': '\u2281', '\u2282': '\u2283', '\u2284': '\u2285',

  284. 		'\u2286': '\u2287', '\u2288': '\u2289', '\u228a': '\u228b',

  285. 		'\u228f': '\u2290', '\u2291': '\u2292', '\u2298': '\u29b8',

  286. 		'\u22a2': '\u22a3', '\u22a6': '\u2ade', '\u22a8': '\u2ae4',

  287. 		'\u22a9': '\u2ae3', '\u22ab': '\u2ae5', '\u22b0': '\u22b1',

  288. 		'\u22b2': '\u22b3', '\u22b4': '\u22b5', '\u22b6': '\u22b7',

  289. 		'\u22c9': '\u22ca', '\u22cb': '\u22cc', '\u22d0': '\u22d1',

  290. 		'\u22d6': '\u22d7', '\u22d8': '\u22d9', '\u22da': '\u22db',

  291. 		'\u22dc': '\u22dd', '\u22de': '\u22df', '\u22e0': '\u22e1',

  292. 		'\u22e2': '\u22e3', '\u22e4': '\u22e5', '\u22e6': '\u22e7',

  293. 		'\u22e8': '\u22e9', '\u22ea': '\u22eb', '\u22ec': '\u22ed',

  294. 		'\u22f0': '\u22f1', '\u22f2': '\u22fa', '\u22f3': '\u22fb',

  295. 		'\u22f4': '\u22fc', '\u22f6': '\u22fd', '\u22f7': '\u22fe',

  296. 		'\u2308': '\u2309', '\u230a': '\u230b', '\u2329': '\u232a',

  297. 		'\u23b4': '\u23b5', '\u2768': '\u2769', '\u276a': '\u276b',

  298. 		'\u276c': '\u276d', '\u276e': '\u276f', '\u2770': '\u2771',

  299. 		'\u2772': '\u2773', '\u2774': '\u2775', '\u27c3': '\u27c4',

  300. 		'\u27c5': '\u27c6', '\u27d5': '\u27d6', '\u27dd': '\u27de',

  301. 		'\u27e2': '\u27e3', '\u27e4': '\u27e5', '\u27e6': '\u27e7',

  302. 		'\u27e8': '\u27e9', '\u27ea': '\u27eb', '\u2983': '\u2984',

  303. 		'\u2985': '\u2986', '\u2987': '\u2988', '\u2989': '\u298a',

  304. 		'\u298b': '\u298c', '\u298d': '\u298e', '\u298f': '\u2990',

  305. 		'\u2991': '\u2992', '\u2993': '\u2994', '\u2995': '\u2996',

  306. 		'\u2997': '\u2998', '\u29c0': '\u29c1', '\u29c4': '\u29c5',

  307. 		'\u29cf': '\u29d0', '\u29d1': '\u29d2', '\u29d4': '\u29d5',

  308. 		'\u29d8': '\u29d9', '\u29da': '\u29db', '\u29f8': '\u29f9',

  309. 		'\u29fc': '\u29fd', '\u2a2b': '\u2a2c', '\u2a2d': '\u2a2e',

  310. 		'\u2a34': '\u2a35', '\u2a3c': '\u2a3d', '\u2a64': '\u2a65',

  311. 		'\u2a79': '\u2a7a', '\u2a7d': '\u2a7e', '\u2a7f': '\u2a80',

  312. 		'\u2a81': '\u2a82', '\u2a83': '\u2a84', '\u2a8b': '\u2a8c',

  313. 		'\u2a91': '\u2a92', '\u2a93': '\u2a94', '\u2a95': '\u2a96',

  314. 		'\u2a97': '\u2a98', '\u2a99': '\u2a9a', '\u2a9b': '\u2a9c',

  315. 		'\u2aa1': '\u2aa2', '\u2aa6': '\u2aa7', '\u2aa8': '\u2aa9',

  316. 		'\u2aaa': '\u2aab', '\u2aac': '\u2aad', '\u2aaf': '\u2ab0',

  317. 		'\u2ab3': '\u2ab4', '\u2abb': '\u2abc', '\u2abd': '\u2abe',

  318. 		'\u2abf': '\u2ac0', '\u2ac1': '\u2ac2', '\u2ac3': '\u2ac4',

  319. 		'\u2ac5': '\u2ac6', '\u2acd': '\u2ace', '\u2acf': '\u2ad0',

  320. 		'\u2ad1': '\u2ad2', '\u2ad3': '\u2ad4', '\u2ad5': '\u2ad6',

  321. 		'\u2aec': '\u2aed', '\u2af7': '\u2af8', '\u2af9': '\u2afa',

  322. 		'\u2e02': '\u2e03', '\u2e04': '\u2e05', '\u2e09': '\u2e0a',

  323. 		'\u2e0c': '\u2e0d', '\u2e1c': '\u2e1d', '\u2e20': '\u2e21',

  324. 		'\u3008': '\u3009', '\u300a': '\u300b', '\u300c': '\u300d',

  325. 		'\u300e': '\u300f', '\u3010': '\u3011', '\u3014': '\u3015',

  326. 		'\u3016': '\u3017', '\u3018': '\u3019', '\u301a': '\u301b',

  327. 		'\u301d': '\u301e', '\ufd3e': '\ufd3f', '\ufe17': '\ufe18',

  328. 		'\ufe35': '\ufe36', '\ufe37': '\ufe38', '\ufe39': '\ufe3a',

  329. 		'\ufe3b': '\ufe3c', '\ufe3d': '\ufe3e', '\ufe3f': '\ufe40',

  330. 		'\ufe41': '\ufe42', '\ufe43': '\ufe44', '\ufe47': '\ufe48',

  331. 		'\ufe59': '\ufe5a', '\ufe5b': '\ufe5c', '\ufe5d': '\ufe5e',

  332. 		'\uff08': '\uff09', '\uff1c': '\uff1e', '\uff3b': '\uff3d',

  333. 		'\uff5b': '\uff5d', '\uff5f': '\uff60', '\uff62': '\uff63',

  334. 	}

  335. 

  336. 	bracketsPattern := `[` + regexp.QuoteMeta(joinRuneMap(brackets)) + `]`

  337. 

  338. 	// Finds opening brackets and their closing counterparts (including pod and heredoc)

  339. 	// and modifies state groups and position accordingly

  340. 	findBrackets := func(tokenClass RakuToken) MutatorFunc {

  341. 		return func(state *LexerState) error {

  342. 			var openingChars []rune

  343. 			var adverbs []rune

  344. 			switch tokenClass {

  345. 			case rakuPod:

  346. 				openingChars = []rune(strings.Join(state.Groups[1:5], ``))

  347. 			default:

  348. 				adverbs = []rune(state.NamedGroups[`adverbs`])

  349. 				openingChars = []rune(state.NamedGroups[`opening_delimiters`])

  350. 			}

  351. 

  352. 			openingChar := openingChars[0]

  353. 

  354. 			nChars := len(openingChars)

  355. 

  356. 			var closingChar rune

  357. 			var closingCharExists bool

  358. 			var closingChars []rune

  359. 

  360. 			switch tokenClass {

  361. 			case rakuPod:

  362. 				closingCharExists = true

  363. 			default:

  364. 				closingChar, closingCharExists = brackets[openingChar]

  365. 			}

  366. 

  367. 			switch tokenClass {

  368. 			case rakuPodFormatter:

  369. 				formatter := StringOther

  370. 

  371. 				switch state.NamedGroups[`keyword`] {

  372. 				case "B":

  373. 					formatter = GenericStrong

  374. 				case "I":

  375. 					formatter = GenericEmph

  376. 				case "U":

  377. 					formatter = GenericUnderline

  378. 				}

  379. 

  380. 				formatterRule := ruleReplacingConfig{

  381. 					pattern:      `.+?`,

  382. 					tokenType:    formatter,

  383. 					mutator:      nil,

  384. 					stateName:    `pod-formatter`,

  385. 					rulePosition: bottomRule,

  386. 				}

  387. 

  388. 				err := replaceRule(formatterRule)(state)

  389. 				if err != nil {

  390. 					panic(err)

  391. 				}

  392. 

  393. 				err = replaceRule(ruleReplacingConfig{

  394. 					delimiter:              []rune{closingChar},

  395. 					tokenType:              Punctuation,

  396. 					stateName:              `pod-formatter`,

  397. 					pushState:              true,

  398. 					numberOfDelimiterChars: nChars,

  399. 					appendMutator:          popRule(formatterRule),

  400. 				})(state)

  401. 				if err != nil {

  402. 					panic(err)

  403. 				}

  404. 

  405. 				return nil

  406. 			case rakuMatchRegex:

  407. 				var delimiter []rune

  408. 				if closingCharExists {

  409. 					delimiter = []rune{closingChar}

  410. 				} else {

  411. 					delimiter = openingChars

  412. 				}

  413. 

  414. 				err := replaceRule(ruleReplacingConfig{

  415. 					delimiter: delimiter,

  416. 					tokenType: Punctuation,

  417. 					stateName: `regex`,

  418. 					popState:  true,

  419. 					pushState: true,

  420. 				})(state)

  421. 				if err != nil {

  422. 					panic(err)

  423. 				}

  424. 

  425. 				return nil

  426. 			case rakuSubstitutionRegex:

  427. 				delimiter := regexp2.Escape(string(openingChars))

  428. 

  429. 				err := replaceRule(ruleReplacingConfig{

  430. 					pattern:      `(` + delimiter + `)` + `((?:\\\\|\\/|.)*?)` + `(` + delimiter + `)`,

  431. 					tokenType:    ByGroups(Punctuation, UsingSelf(`qq`), Punctuation),

  432. 					rulePosition: topRule,

  433. 					stateName:    `regex`,

  434. 					popState:     true,

  435. 					pushState:    true,

  436. 				})(state)

  437. 				if err != nil {

  438. 					panic(err)

  439. 				}

  440. 

  441. 				return nil

  442. 			}

  443. 

  444. 			text := state.Text

  445. 

  446. 			var endPos int

  447. 

  448. 			var nonMirroredOpeningCharPosition int

  449. 

  450. 			if !closingCharExists {

  451. 				// it's not a mirrored character, which means we

  452. 				// just need to look for the next occurrence

  453. 				closingChars = openingChars

  454. 				nonMirroredOpeningCharPosition = indexAt(text, closingChars, state.Pos)

  455. 				endPos = nonMirroredOpeningCharPosition

  456. 			} else {

  457. 				var podRegex *regexp2.Regexp

  458. 				if tokenClass == rakuPod {

  459. 					podRegex = regexp2.MustCompile(

  460. 						state.NamedGroups[`ws`]+`=end`+`\s+`+regexp2.Escape(state.NamedGroups[`name`]),

  461. 						0,

  462. 					)

  463. 				} else {

  464. 					closingChars = []rune(strings.Repeat(string(closingChar), nChars))

  465. 				}

  466. 

  467. 				// we need to look for the corresponding closing character,

  468. 				// keep nesting in mind

  469. 				nestingLevel := 1

  470. 

  471. 				searchPos := state.Pos - nChars

  472. 

  473. 				var nextClosePos int

  474. 

  475. 				for nestingLevel > 0 {

  476. 					if tokenClass == rakuPod {

  477. 						match, err := podRegex.FindRunesMatchStartingAt(text, searchPos+nChars)

  478. 						if err == nil {

  479. 							closingChars = match.Runes()

  480. 							nextClosePos = match.Index

  481. 						} else {

  482. 							nextClosePos = -1

  483. 						}

  484. 					} else {

  485. 						nextClosePos = indexAt(text, closingChars, searchPos+nChars)

  486. 					}

  487. 

  488. 					nextOpenPos := indexAt(text, openingChars, searchPos+nChars)

  489. 

  490. 					switch {

  491. 					case nextClosePos == -1:

  492. 						nextClosePos = len(text)

  493. 						nestingLevel = 0

  494. 					case nextOpenPos != -1 && nextOpenPos < nextClosePos:

  495. 						nestingLevel++

  496. 						nChars = len(openingChars)

  497. 						searchPos = nextOpenPos

  498. 					default: // next_close_pos < next_open_pos

  499. 						nestingLevel--

  500. 						nChars = len(closingChars)

  501. 						searchPos = nextClosePos

  502. 					}

  503. 				}

  504. 

  505. 				endPos = nextClosePos

  506. 			}

  507. 

  508. 			if endPos < 0 {

  509. 				// if we didn't find a closer, just highlight the

  510. 				// rest of the text in this class

  511. 				endPos = len(text)

  512. 			}

  513. 

  514. 			adverbre := regexp.MustCompile(`:to\b|:heredoc\b`)

  515. 			var heredocTerminator []rune

  516. 			var endHeredocPos int

  517. 			if adverbre.MatchString(string(adverbs)) {

  518. 				if endPos != len(text) {

  519. 					heredocTerminator = text[state.Pos:endPos]

  520. 					nChars = len(heredocTerminator)

  521. 				} else {

  522. 					endPos = state.Pos + 1

  523. 					heredocTerminator = []rune{}

  524. 					nChars = 0

  525. 				}

  526. 

  527. 				if nChars > 0 {

  528. 					endHeredocPos = indexAt(text[endPos:], heredocTerminator, 0)

  529. 					if endHeredocPos > -1 {

  530. 						endPos += endHeredocPos

  531. 					} else {

  532. 						endPos = len(text)

  533. 					}

  534. 				}

  535. 			}

  536. 

  537. 			textBetweenBrackets := string(text[state.Pos:endPos])

  538. 			switch tokenClass {

  539. 			case rakuPod, rakuPodDeclaration, rakuNameAttribute:

  540. 				state.NamedGroups[`value`] = textBetweenBrackets

  541. 				state.NamedGroups[`closing_delimiters`] = string(closingChars)

  542. 			case rakuQuote:

  543. 				if len(heredocTerminator) > 0 {

  544. 					// Length of heredoc terminator + closing chars + `;`

  545. 					heredocFristPunctuationLen := nChars + len(openingChars) + 1

  546. 

  547. 					state.NamedGroups[`opening_delimiters`] = string(openingChars) +

  548. 						string(text[state.Pos:state.Pos+heredocFristPunctuationLen])

  549. 

  550. 					state.NamedGroups[`value`] =

  551. 						string(text[state.Pos+heredocFristPunctuationLen : endPos])

  552. 

  553. 					if endHeredocPos > -1 {

  554. 						state.NamedGroups[`closing_delimiters`] = string(heredocTerminator)

  555. 					}

  556. 				} else {

  557. 					state.NamedGroups[`value`] = textBetweenBrackets

  558. 					if nChars > 0 {

  559. 						state.NamedGroups[`closing_delimiters`] = string(closingChars)

  560. 					}

  561. 				}

  562. 			default:

  563. 				state.Groups = []string{state.Groups[0] + string(text[state.Pos:endPos+nChars])}

  564. 			}

  565. 

  566. 			state.Pos = endPos + nChars

  567. 

  568. 			return nil

  569. 		}

  570. 	}

  571. 

  572. 	// Raku rules

  573. 	// Empty capture groups are placeholders and will be replaced by mutators

  574. 	// DO NOT REMOVE THEM!

  575. 	return Rules{

  576. 		"root": {

  577. 			// Placeholder, will be overwritten by mutators, DO NOT REMOVE!

  578. 			{`\A\z`, nil, nil},

  579. 			Include("common"),

  580. 			{`{`, Punctuation, Push(`root`)},

  581. 			{`\(`, Punctuation, Push(`root`)},

  582. 			{`[)}]`, Punctuation, Pop(1)},

  583. 			{`;`, Punctuation, nil},

  584. 			{`\[|\]`, Operator, nil},

  585. 			{`.+?`, Text, nil},

  586. 		},

  587. 		"common": {

  588. 			{`^#![^\n]*$`, CommentHashbang, nil},

  589. 			Include("pod"),

  590. 			// Multi-line, Embedded comment

  591. 			{

  592. 				"#`(?<opening_delimiters>(?<delimiter>" + bracketsPattern + `)\k<delimiter>*)`,

  593. 				CommentMultiline,

  594. 				findBrackets(rakuMultilineComment),

  595. 			},

  596. 			{`#[^\n]*$`, CommentSingle, nil},

  597. 			// /regex/

  598. 			{

  599. 				`(?<=(?:^|\(|=|:|~~|\[|{|,|=>)\s*)(/)(?!\]|\))((?:\\\\|\\/|.)*?)((?<!(?<!\\)\\)/(?!'|"))`,

  600. 				ByGroups(Punctuation, UsingSelf("regex"), Punctuation),

  601. 				nil,

  602. 			},

  603. 			Include("variable"),

  604. 			// ::?VARIABLE

  605. 			{`::\?\w+(?::[_UD])?`, NameVariableGlobal, nil},

  606. 			// Version

  607. 			{

  608. 				`\b(v)(\d+)((?:\.(?:\*|[\d\w]+))*)(\+)?`,

  609. 				ByGroups(Keyword, NumberInteger, NameEntity, Operator),

  610. 				nil,

  611. 			},

  612. 			Include("number"),

  613. 			// Hyperoperator | »*«

  614. 			{`(>>)(\S+?)(<<)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  615. 			{`(»)(\S+?)(«)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  616. 			// Hyperoperator | «*«

  617. 			{`(<<)(\S+?)(<<)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  618. 			{`(«)(\S+?)(«)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  619. 			// Hyperoperator | »*»

  620. 			{`(>>)(\S+?)(>>)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  621. 			{`(»)(\S+?)(»)`, ByGroups(Operator, UsingSelf("root"), Operator), nil},

  622. 			// <<quoted words>>

  623. 			{`(?<!(?:\d+|\.(?:Int|Numeric)|[$@%]\*?[\w':-]+\s+|[\])}]\s+)\s*)(<<)(?!(?:(?!>>)[^\n])+?[},;] *\n)(?!(?:(?!>>).)+?>>\S+?>>)`, Punctuation, Push("<<")},

  624. 			// «quoted words»

  625. 			{`(?<!(?:\d+|\.(?:Int|Numeric)|[$@%]\*?[\w':-]+\s+|[\])}]\s+)\s*)(«)(?![^»]+?[},;] *\n)(?![^»]+?»\S+?»)`, Punctuation, Push("«")},

  626. 			// [<]

  627. 			{`(?<=\[\\?)<(?=\])`, Operator, nil},

  628. 			// < and > operators | something < onething > something

  629. 			{

  630. 				`(?<=[$@%&]?\w[\w':-]* +)(<=?)( *[^ ]+? *)(>=?)(?= *[$@%&]?\w[\w':-]*)`,

  631. 				ByGroups(Operator, UsingSelf("root"), Operator),

  632. 				nil,

  633. 			},

  634. 			// <quoted words>

  635. 			{

  636. 				`(?<!(?:\d+|\.(?:Int|Numeric)|[$@%]\*?[\w':-]+\s+|[\])}]\s+)\s*)(<)((?:(?![,;)}] *(?:#[^\n]+)?\n)[^<>])+?)(>)(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%]\*?\w[\w':-]*[^(]|\s+\[))`,

  637. 				ByGroups(Punctuation, String, Punctuation),

  638. 				nil,

  639. 			},

  640. 			{`C?X::['\w:-]+`, NameException, nil},

  641. 			Include("metaoperator"),

  642. 			// Pair | key => value

  643. 			{

  644. 				`(\w[\w'-]*)(\s*)(=>)`,

  645. 				ByGroups(String, Text, Operator),

  646. 				nil,

  647. 			},

  648. 			Include("colon-pair"),

  649. 			// Token

  650. 			{

  651. 				`(?<=(?:^|\s)(?:regex|token|rule)(\s+))` + namePattern + colonPairLookahead + `\s*[({])`,

  652. 				NameFunction,

  653. 				Push("token", "name-adverb"),

  654. 			},

  655. 			// Substitution

  656. 			{`(?<=^|\b|\s)(?<!\.)(ss|S|s|TR|tr)\b(\s*)`, ByGroups(Keyword, Text), Push("substitution")},

  657. 			{keywordsPattern, Keyword, nil},

  658. 			{builtinTypesPattern, KeywordType, nil},

  659. 			{builtinRoutinesPattern, NameBuiltin, nil},

  660. 			// Class name

  661. 			{

  662. 				`(?<=(?:^|\s)(?:class|grammar|role|does|but|is|subset|of)\s+)` + namePattern,

  663. 				NameClass,

  664. 				Push("name-adverb"),

  665. 			},

  666. 			//  Routine

  667. 			{

  668. 				`(?<=(?:^|\s)(?:sub|method|multi sub|multi)\s+)!?` + namePattern + colonPairLookahead + `\s*[({])`,

  669. 				NameFunction,

  670. 				Push("name-adverb"),

  671. 			},

  672. 			// Constant

  673. 			{`(?<=\bconstant\s+)` + namePattern, NameConstant, Push("name-adverb")},

  674. 			// Namespace

  675. 			{`(?<=\b(?:use|module|package)\s+)` + namePattern, NameNamespace, Push("name-adverb")},

  676. 			Include("operator"),

  677. 			Include("single-quote"),

  678. 			{`(?<!(?<!\\)\\)"`, Punctuation, Push("double-quotes")},

  679. 			// m,rx regex

  680. 			{`(?<=^|\b|\s)(ms|m|rx)\b(\s*)`, ByGroups(Keyword, Text), Push("rx")},

  681. 			// Quote constructs

  682. 			{

  683. 				`(?<=^|\b|\s)(?<keyword>(?:qq|q|Q))(?<adverbs>(?::?(?:heredoc|to|qq|ww|q|w|s|a|h|f|c|b|to|v|x))*)(?<ws>\s*)(?<opening_delimiters>(?<delimiter>[^0-9a-zA-Z:\s])\k<delimiter>*)`,

  684. 				EmitterFunc(quote),

  685. 				findBrackets(rakuQuote),

  686. 			},

  687. 			// Function

  688. 			{

  689. 				`\b` + namePattern + colonPairLookahead + `\()`,

  690. 				NameFunction,

  691. 				Push("name-adverb"),

  692. 			},

  693. 			// Method

  694. 			{

  695. 				`(?<!\.\.[?^*+]?)(?<=(?:\.[?^*+&]?)|self!)` + namePattern + colonPairLookahead + `\b)`,

  696. 				NameFunction,

  697. 				Push("name-adverb"),

  698. 			},

  699. 			// Indirect invocant

  700. 			{namePattern + `(?=\s+\W?['\w:-]+:\W)`, NameFunction, Push("name-adverb")},

  701. 			{`(?<=\W)(?:∅|i|e|𝑒|tau|τ|pi|π|Inf|∞)(?=\W)`, NameConstant, nil},

  702. 			{`(｢)([^｣]*)(｣)`, ByGroups(Punctuation, String, Punctuation), nil},

  703. 			{`(?<=^ *)\b` + namePattern + `(?=:\s*(?:for|while|loop))`, NameLabel, nil},

  704. 			// Sigilless variable

  705. 			{

  706. 				`(?<=\b(?:my|our|constant|let|temp)\s+)\\` + namePattern,

  707. 				NameVariable,

  708. 				Push("name-adverb"),

  709. 			},

  710. 			{namePattern, Name, Push("name-adverb")},

  711. 		},

  712. 		"rx": {

  713. 			Include("colon-pair-attribute"),

  714. 			{

  715. 				`(?<opening_delimiters>(?<delimiter>[^\w:\s])\k<delimiter>*)`,

  716. 				ByGroupNames(

  717. 					map[string]Emitter{

  718. 						`opening_delimiters`: Punctuation,

  719. 						`delimiter`:          nil,

  720. 					},

  721. 				),

  722. 				findBrackets(rakuMatchRegex),

  723. 			},

  724. 		},

  725. 		"substitution": {

  726. 			Include("colon-pair-attribute"),

  727. 			// Substitution | s{regex} = value

  728. 			{

  729. 				`(?<opening_delimiters>(?<delimiter>` + bracketsPattern + `)\k<delimiter>*)`,

  730. 				ByGroupNames(map[string]Emitter{

  731. 					`opening_delimiters`: Punctuation,

  732. 					`delimiter`:          nil,

  733. 				}),

  734. 				findBrackets(rakuMatchRegex),

  735. 			},

  736. 			// Substitution | s/regex/string/

  737. 			{

  738. 				`(?<opening_delimiters>[^\w:\s])`,

  739. 				Punctuation,

  740. 				findBrackets(rakuSubstitutionRegex),

  741. 			},

  742. 		},

  743. 		"number": {

  744. 			{`0_?[0-7]+(_[0-7]+)*`, LiteralNumberOct, nil},

  745. 			{`0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*`, LiteralNumberHex, nil},

  746. 			{`0b[01]+(_[01]+)*`, LiteralNumberBin, nil},

  747. 			{

  748. 				`(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?`,

  749. 				LiteralNumberFloat,

  750. 				nil,

  751. 			},

  752. 			{`(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*`, LiteralNumberFloat, nil},

  753. 			{`(?<=\d+)i`, NameConstant, nil},

  754. 			{`\d+(_\d+)*`, LiteralNumberInteger, nil},

  755. 		},

  756. 		"name-adverb": {

  757. 			Include("colon-pair-attribute-keyvalue"),

  758. 			Default(Pop(1)),

  759. 		},

  760. 		"colon-pair": {

  761. 			// :key(value)

  762. 			{colonPairPattern, colonPair(String), findBrackets(rakuNameAttribute)},

  763. 			// :123abc

  764. 			{

  765. 				`(:)(\d+)(\w[\w'-]*)`,

  766. 				ByGroups(Punctuation, UsingSelf("number"), String),

  767. 				nil,

  768. 			},

  769. 			// :key

  770. 			{`(:)(!?)(\w[\w'-]*)`, ByGroups(Punctuation, Operator, String), nil},

  771. 			{`\s+`, Text, nil},

  772. 		},

  773. 		"colon-pair-attribute": {

  774. 			// :key(value)

  775. 			{colonPairPattern, colonPair(NameAttribute), findBrackets(rakuNameAttribute)},

  776. 			// :123abc

  777. 			{

  778. 				`(:)(\d+)(\w[\w'-]*)`,

  779. 				ByGroups(Punctuation, UsingSelf("number"), NameAttribute),

  780. 				nil,

  781. 			},

  782. 			// :key

  783. 			{`(:)(!?)(\w[\w'-]*)`, ByGroups(Punctuation, Operator, NameAttribute), nil},

  784. 			{`\s+`, Text, nil},

  785. 		},

  786. 		"colon-pair-attribute-keyvalue": {

  787. 			// :key(value)

  788. 			{colonPairPattern, colonPair(NameAttribute), findBrackets(rakuNameAttribute)},

  789. 		},

  790. 		"escape-qq": {

  791. 			{

  792. 				`(?<!(?<!\\)\\)(\\qq)(\[)(.+?)(\])`,

  793. 				ByGroups(StringEscape, Punctuation, UsingSelf("qq"), Punctuation),

  794. 				nil,

  795. 			},

  796. 		},

  797. 		`escape-char`: {

  798. 			{`(?<!(?<!\\)\\)(\\[abfrnrt])`, StringEscape, nil},

  799. 		},

  800. 		`escape-single-quote`: {

  801. 			{`(?<!(?<!\\)\\)(\\)(['\\])`, ByGroups(StringEscape, StringSingle), nil},

  802. 		},

  803. 		"escape-c-name": {

  804. 			{

  805. 				`(?<!(?<!\\)\\)(\\[c|C])(\[)(.+?)(\])`,

  806. 				ByGroups(StringEscape, Punctuation, String, Punctuation),

  807. 				nil,

  808. 			},

  809. 		},

  810. 		"escape-hexadecimal": {

  811. 			{

  812. 				`(?<!(?<!\\)\\)(\\[x|X])(\[)([0-9a-fA-F]+)(\])`,

  813. 				ByGroups(StringEscape, Punctuation, NumberHex, Punctuation),

  814. 				nil,

  815. 			},

  816. 			{`(\\[x|X])([0-9a-fA-F]+)`, ByGroups(StringEscape, NumberHex), nil},

  817. 		},

  818. 		"regex": {

  819. 			// Placeholder, will be overwritten by mutators, DO NOT REMOVE!

  820. 			{`\A\z`, nil, nil},

  821. 			Include("regex-escape-class"),

  822. 			Include(`regex-character-escape`),

  823. 			// $(code)

  824. 			{

  825. 				`([$@])((?<!(?<!\\)\\)\()`,

  826. 				ByGroups(Keyword, Punctuation),

  827. 				replaceRule(ruleReplacingConfig{

  828. 					delimiter: []rune(`)`),

  829. 					tokenType: Punctuation,

  830. 					stateName: `root`,

  831. 					pushState: true,

  832. 				}),

  833. 			},

  834. 			// Exclude $/ from variables, because we can't get out of the end of the slash regex: $/;

  835. 			{`\$(?=/)`, NameEntity, nil},

  836. 			// Exclude $ from variables

  837. 			{`\$(?=\z|\s|[^<(\w*!.])`, NameEntity, nil},

  838. 			Include("variable"),

  839. 			Include("escape-c-name"),

  840. 			Include("escape-hexadecimal"),

  841. 			Include("number"),

  842. 			Include("single-quote"),

  843. 			// :my variable code ...

  844. 			{

  845. 				`(?<!(?<!\\)\\)(:)(my|our|state|constant|temp|let)`,

  846. 				ByGroups(Operator, KeywordDeclaration),

  847. 				replaceRule(ruleReplacingConfig{

  848. 					delimiter: []rune(`;`),

  849. 					tokenType: Punctuation,

  850. 					stateName: `root`,

  851. 					pushState: true,

  852. 				}),

  853. 			},

  854. 			// <{code}>

  855. 			{

  856. 				`(?<!(?<!\\)\\)(<)([?!.]*)((?<!(?<!\\)\\){)`,

  857. 				ByGroups(Punctuation, Operator, Punctuation),

  858. 				replaceRule(ruleReplacingConfig{

  859. 					delimiter: []rune(`}>`),

  860. 					tokenType: Punctuation,

  861. 					stateName: `root`,

  862. 					pushState: true,

  863. 				}),

  864. 			},

  865. 			// {code}

  866. 			Include(`closure`),

  867. 			// Properties

  868. 			{`(:)(\w+)`, ByGroups(Punctuation, NameAttribute), nil},

  869. 			// Operator

  870. 			{`\|\||\||&&|&|\.\.|\*\*|%%|%|:|!|<<|«|>>|»|\+|\*\*|\*|\?|=|~|<~~>`, Operator, nil},

  871. 			// Anchors

  872. 			{`\^\^|\^|\$\$|\$`, NameEntity, nil},

  873. 			{`\.`, NameEntity, nil},

  874. 			{`#[^\n]*\n`, CommentSingle, nil},

  875. 			// Lookaround

  876. 			{

  877. 				`(?<!(?<!\\)\\)(<)(\s*)([?!.]+)(\s*)(after|before)`,

  878. 				ByGroups(Punctuation, Text, Operator, Text, OperatorWord),

  879. 				replaceRule(ruleReplacingConfig{

  880. 					delimiter: []rune(`>`),

  881. 					tokenType: Punctuation,

  882. 					stateName: `regex`,

  883. 					pushState: true,

  884. 				}),

  885. 			},

  886. 			{

  887. 				`(?<!(?<!\\)\\)(<)([|!?.]*)(wb|ww|ws|w)(>)`,

  888. 				ByGroups(Punctuation, Operator, OperatorWord, Punctuation),

  889. 				nil,

  890. 			},

  891. 			// <$variable>

  892. 			{

  893. 				`(?<!(?<!\\)\\)(<)([?!.]*)([$@]\w[\w:-]*)(>)`,

  894. 				ByGroups(Punctuation, Operator, NameVariable, Punctuation),

  895. 				nil,

  896. 			},

  897. 			// Capture markers

  898. 			{`(?<!(?<!\\)\\)<\(|\)>`, Operator, nil},

  899. 			{

  900. 				`(?<!(?<!\\)\\)(<)(\w[\w:-]*)(=\.?)`,

  901. 				ByGroups(Punctuation, NameVariable, Operator),

  902. 				Push(`regex-variable`),

  903. 			},

  904. 			{

  905. 				`(?<!(?<!\\)\\)(<)([|!?.&]*)(\w(?:(?!:\s)[\w':-])*)`,

  906. 				ByGroups(Punctuation, Operator, NameFunction),

  907. 				Push(`regex-function`),

  908. 			},

  909. 			{`(?<!(?<!\\)\\)<`, Punctuation, Push("regex-property")},

  910. 			{`(?<!(?<!\\)\\)"`, Punctuation, Push("double-quotes")},

  911. 			{`(?<!(?<!\\)\\)(?:\]|\))`, Punctuation, Pop(1)},

  912. 			{`(?<!(?<!\\)\\)(?:\[|\()`, Punctuation, Push("regex")},

  913. 			{`.+?`, StringRegex, nil},

  914. 		},

  915. 		"regex-class-builtin": {

  916. 			{

  917. 				`\b(?:alnum|alpha|blank|cntrl|digit|graph|lower|print|punct|space|upper|xdigit|same|ident)\b`,

  918. 				NameBuiltin,

  919. 				nil,

  920. 			},

  921. 		},

  922. 		"regex-function": {

  923. 			// <function>

  924. 			{`(?<!(?<!\\)\\)>`, Punctuation, Pop(1)},

  925. 			// <function(parameter)>

  926. 			{

  927. 				`\(`,

  928. 				Punctuation,

  929. 				replaceRule(ruleReplacingConfig{

  930. 					delimiter: []rune(`)>`),

  931. 					tokenType: Punctuation,

  932. 					stateName: `root`,

  933. 					popState:  true,

  934. 					pushState: true,

  935. 				}),

  936. 			},

  937. 			// <function value>

  938. 			{

  939. 				`\s+`,

  940. 				StringRegex,

  941. 				replaceRule(ruleReplacingConfig{

  942. 					delimiter: []rune(`>`),

  943. 					tokenType: Punctuation,

  944. 					stateName: `regex`,

  945. 					popState:  true,

  946. 					pushState: true,

  947. 				}),

  948. 			},

  949. 			// <function: value>

  950. 			{

  951. 				`:`,

  952. 				Punctuation,

  953. 				replaceRule(ruleReplacingConfig{

  954. 					delimiter: []rune(`>`),

  955. 					tokenType: Punctuation,

  956. 					stateName: `root`,

  957. 					popState:  true,

  958. 					pushState: true,

  959. 				}),

  960. 			},

  961. 		},

  962. 		"regex-variable": {

  963. 			Include(`regex-starting-operators`),

  964. 			// <var=function(

  965. 			{

  966. 				`(&)?(\w(?:(?!:\s)[\w':-])*)(?=\()`,

  967. 				ByGroups(Operator, NameFunction),

  968. 				Mutators(Pop(1), Push(`regex-function`)),

  969. 			},

  970. 			// <var=function>

  971. 			{`(&)?(\w[\w':-]*)(>)`, ByGroups(Operator, NameFunction, Punctuation), Pop(1)},

  972. 			// <var=

  973. 			Default(Pop(1), Push(`regex-property`)),

  974. 		},

  975. 		"regex-property": {

  976. 			{`(?<!(?<!\\)\\)>`, Punctuation, Pop(1)},

  977. 			Include("regex-class-builtin"),

  978. 			Include("variable"),

  979. 			Include(`regex-starting-operators`),

  980. 			Include("colon-pair-attribute"),

  981. 			{`(?<!(?<!\\)\\)\[`, Punctuation, Push("regex-character-class")},

  982. 			{`\+|\-`, Operator, nil},

  983. 			{`@[\w':-]+`, NameVariable, nil},

  984. 			{`.+?`, StringRegex, nil},

  985. 		},

  986. 		`regex-starting-operators`: {

  987. 			{`(?<=<)[|!?.]+`, Operator, nil},

  988. 		},

  989. 		"regex-escape-class": {

  990. 			{`(?i)\\n|\\t|\\h|\\v|\\s|\\d|\\w`, StringEscape, nil},

  991. 		},

  992. 		`regex-character-escape`: {

  993. 			{`(?<!(?<!\\)\\)(\\)(.)`, ByGroups(StringEscape, StringRegex), nil},

  994. 		},

  995. 		"regex-character-class": {

  996. 			{`(?<!(?<!\\)\\)\]`, Punctuation, Pop(1)},

  997. 			Include("regex-escape-class"),

  998. 			Include("escape-c-name"),

  999. 			Include("escape-hexadecimal"),

  1000. 			Include(`regex-character-escape`),

  1001. 			Include("number"),

  1002. 			{`\.\.`, Operator, nil},

  1003. 			{`.+?`, StringRegex, nil},

  1004. 		},

  1005. 		"metaoperator": {

  1006. 			// Z[=>]

  1007. 			{

  1008. 				`\b([RZX]+)\b(\[)([^\s\]]+?)(\])`,

  1009. 				ByGroups(OperatorWord, Punctuation, UsingSelf("root"), Punctuation),

  1010. 				nil,

  1011. 			},

  1012. 			// Z=>

  1013. 			{`\b([RZX]+)\b([^\s\]]+)`, ByGroups(OperatorWord, UsingSelf("operator")), nil},

  1014. 		},

  1015. 		"operator": {

  1016. 			// Word Operator

  1017. 			{wordOperatorsPattern, OperatorWord, nil},

  1018. 			// Operator

  1019. 			{operatorsPattern, Operator, nil},

  1020. 		},

  1021. 		"pod": {

  1022. 			// Single-line pod declaration

  1023. 			{`(#[|=])\s`, Keyword, Push("pod-single")},

  1024. 			// Multi-line pod declaration

  1025. 			{

  1026. 				"(?<keyword>#[|=])(?<opening_delimiters>(?<delimiter>" + bracketsPattern + `)\k<delimiter>*)(?<value>)(?<closing_delimiters>)`,

  1027. 				ByGroupNames(

  1028. 					map[string]Emitter{

  1029. 						`keyword`:            Keyword,

  1030. 						`opening_delimiters`: Punctuation,

  1031. 						`delimiter`:          nil,

  1032. 						`value`:              UsingSelf("pod-declaration"),

  1033. 						`closing_delimiters`: Punctuation,

  1034. 					}),

  1035. 				findBrackets(rakuPodDeclaration),

  1036. 			},

  1037. 			Include("pod-blocks"),

  1038. 		},

  1039. 		"pod-blocks": {

  1040. 			// =begin code

  1041. 			{

  1042. 				`(?<=^ *)(?<ws> *)(?<keyword>=begin)(?<ws2> +)(?<name>code)(?<config>[^\n]*)(?<value>.*?)(?<ws3>^\k<ws>)(?<end_keyword>=end)(?<ws4> +)\k<name>`,

  1043. 				EmitterFunc(podCode),

  1044. 				nil,

  1045. 			},

  1046. 			// =begin

  1047. 			{

  1048. 				`(?<=^ *)(?<ws> *)(?<keyword>=begin)(?<ws2> +)(?!code)(?<name>\w[\w'-]*)(?<config>[^\n]*)(?<value>)(?<closing_delimiters>)`,

  1049. 				ByGroupNames(

  1050. 					map[string]Emitter{

  1051. 						`ws`:                 Comment,

  1052. 						`keyword`:            Keyword,

  1053. 						`ws2`:                StringDoc,

  1054. 						`name`:               Keyword,

  1055. 						`config`:             EmitterFunc(podConfig),

  1056. 						`value`:              UsingSelf("pod-begin"),

  1057. 						`closing_delimiters`: Keyword,

  1058. 					}),

  1059. 				findBrackets(rakuPod),

  1060. 			},

  1061. 			// =for ...

  1062. 			{

  1063. 				`(?<=^ *)(?<ws> *)(?<keyword>=(?:for|defn))(?<ws2> +)(?<name>\w[\w'-]*)(?<config>[^\n]*\n)`,

  1064. 				ByGroups(Comment, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)),

  1065. 				Push("pod-paragraph"),

  1066. 			},

  1067. 			// =config

  1068. 			{

  1069. 				`(?<=^ *)(?<ws> *)(?<keyword>=config)(?<ws2> +)(?<name>\w[\w'-]*)(?<config>[^\n]*\n)`,

  1070. 				ByGroups(Comment, Keyword, StringDoc, Keyword, EmitterFunc(podConfig)),

  1071. 				nil,

  1072. 			},

  1073. 			// =alias

  1074. 			{

  1075. 				`(?<=^ *)(?<ws> *)(?<keyword>=alias)(?<ws2> +)(?<name>\w[\w'-]*)(?<value>[^\n]*\n)`,

  1076. 				ByGroups(Comment, Keyword, StringDoc, Keyword, StringDoc),

  1077. 				nil,

  1078. 			},

  1079. 			// =encoding

  1080. 			{

  1081. 				`(?<=^ *)(?<ws> *)(?<keyword>=encoding)(?<ws2> +)(?<name>[^\n]+)`,

  1082. 				ByGroups(Comment, Keyword, StringDoc, Name),

  1083. 				nil,

  1084. 			},

  1085. 			// =para ...

  1086. 			{

  1087. 				`(?<=^ *)(?<ws> *)(?<keyword>=(?:para|table|pod))(?<config>(?<!\n\s*)[^\n]*\n)`,

  1088. 				ByGroups(Comment, Keyword, EmitterFunc(podConfig)),

  1089. 				Push("pod-paragraph"),

  1090. 			},

  1091. 			// =head1 ...

  1092. 			{

  1093. 				`(?<=^ *)(?<ws> *)(?<keyword>=head\d+)(?<ws2> *)(?<config>#?)`,

  1094. 				ByGroups(Comment, Keyword, GenericHeading, Keyword),

  1095. 				Push("pod-heading"),

  1096. 			},

  1097. 			// =item ...

  1098. 			{

  1099. 				`(?<=^ *)(?<ws> *)(?<keyword>=(?:item\d*|comment|data|[A-Z]+))(?<ws2> *)(?<config>#?)`,

  1100. 				ByGroups(Comment, Keyword, StringDoc, Keyword),

  1101. 				Push("pod-paragraph"),

  1102. 			},

  1103. 			{

  1104. 				`(?<=^ *)(?<ws> *)(?<keyword>=finish)(?<config>[^\n]*)`,

  1105. 				ByGroups(Comment, Keyword, EmitterFunc(podConfig)),

  1106. 				Push("pod-finish"),

  1107. 			},

  1108. 			// ={custom} ...

  1109. 			{

  1110. 				`(?<=^ *)(?<ws> *)(?<name>=\w[\w'-]*)(?<ws2> *)(?<config>#?)`,

  1111. 				ByGroups(Comment, Name, StringDoc, Keyword),

  1112. 				Push("pod-paragraph"),

  1113. 			},

  1114. 			// = podconfig

  1115. 			{

  1116. 				`(?<=^ *)(?<keyword> *=)(?<ws> *)(?<config>(?::\w[\w'-]*(?:` + colonPairOpeningBrackets + `.+?` +

  1117. 					colonPairClosingBrackets + `) *)*\n)`,

  1118. 				ByGroups(Keyword, StringDoc, EmitterFunc(podConfig)),

  1119. 				nil,

  1120. 			},

  1121. 		},

  1122. 		"pod-begin": {

  1123. 			Include("pod-blocks"),

  1124. 			Include("pre-pod-formatter"),

  1125. 			{`.+?`, StringDoc, nil},

  1126. 		},

  1127. 		"pod-declaration": {

  1128. 			Include("pre-pod-formatter"),

  1129. 			{`.+?`, StringDoc, nil},

  1130. 		},

  1131. 		"pod-paragraph": {

  1132. 			{`\n *\n|\n(?=^ *=)`, StringDoc, Pop(1)},

  1133. 			Include("pre-pod-formatter"),

  1134. 			{`.+?`, StringDoc, nil},

  1135. 		},

  1136. 		"pod-single": {

  1137. 			{`\n`, StringDoc, Pop(1)},

  1138. 			Include("pre-pod-formatter"),

  1139. 			{`.+?`, StringDoc, nil},

  1140. 		},

  1141. 		"pod-heading": {

  1142. 			{`\n *\n|\n(?=^ *=)`, GenericHeading, Pop(1)},

  1143. 			Include("pre-pod-formatter"),

  1144. 			{`.+?`, GenericHeading, nil},

  1145. 		},

  1146. 		"pod-finish": {

  1147. 			{`\z`, nil, Pop(1)},

  1148. 			Include("pre-pod-formatter"),

  1149. 			{`.+?`, StringDoc, nil},

  1150. 		},

  1151. 		"pre-pod-formatter": {

  1152. 			// C<code>, B<bold>, ...

  1153. 			{

  1154. 				`(?<keyword>[CBIUDTKRPAELZVMSXN])(?<opening_delimiters><+|«)`,

  1155. 				ByGroups(Keyword, Punctuation),

  1156. 				findBrackets(rakuPodFormatter),

  1157. 			},

  1158. 		},

  1159. 		"pod-formatter": {

  1160. 			// Placeholder rule, will be replaced by mutators. DO NOT REMOVE!

  1161. 			{`>`, Punctuation, Pop(1)},

  1162. 			Include("pre-pod-formatter"),

  1163. 			// Placeholder rule, will be replaced by mutators. DO NOT REMOVE!

  1164. 			{`.+?`, StringOther, nil},

  1165. 		},

  1166. 		"variable": {

  1167. 			{variablePattern, NameVariable, Push("name-adverb")},

  1168. 			{globalVariablePattern, NameVariableGlobal, Push("name-adverb")},

  1169. 			{`[$@]<[^>]+>`, NameVariable, nil},

  1170. 			{`\$[/!¢]`, NameVariable, nil},

  1171. 			{`[$@%]`, NameVariable, nil},

  1172. 		},

  1173. 		"single-quote": {

  1174. 			{`(?<!(?<!\\)\\)'`, Punctuation, Push("single-quote-inner")},

  1175. 		},

  1176. 		"single-quote-inner": {

  1177. 			{`(?<!(?<!(?<!\\)\\)\\)'`, Punctuation, Pop(1)},

  1178. 			Include("escape-single-quote"),

  1179. 			Include("escape-qq"),

  1180. 			{`(?:\\\\|\\[^\\]|[^'\\])+?`, StringSingle, nil},

  1181. 		},

  1182. 		"double-quotes": {

  1183. 			{`(?<!(?<!\\)\\)"`, Punctuation, Pop(1)},

  1184. 			Include("qq"),

  1185. 		},

  1186. 		"<<": {

  1187. 			{`>>(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%]\*?[\w':-]+|\s+\[))`, Punctuation, Pop(1)},

  1188. 			Include("ww"),

  1189. 		},

  1190. 		"«": {

  1191. 			{`»(?!\s*(?:\d+|\.(?:Int|Numeric)|[$@%]\*?[\w':-]+|\s+\[))`, Punctuation, Pop(1)},

  1192. 			Include("ww"),

  1193. 		},

  1194. 		"ww": {

  1195. 			Include("single-quote"),

  1196. 			Include("qq"),

  1197. 		},

  1198. 		"qq": {

  1199. 			Include("qq-variable"),

  1200. 			Include("closure"),

  1201. 			Include(`escape-char`),

  1202. 			Include("escape-hexadecimal"),

  1203. 			Include("escape-c-name"),

  1204. 			Include("escape-qq"),

  1205. 			{`.+?`, StringDouble, nil},

  1206. 		},

  1207. 		"qq-variable": {

  1208. 			{

  1209. 				`(?<!(?<!\\)\\)(?:` + variablePattern + `|` + globalVariablePattern + `)` + colonPairLookahead + `)`,

  1210. 				NameVariable,

  1211. 				Push("qq-variable-extras", "name-adverb"),

  1212. 			},

  1213. 		},

  1214. 		"qq-variable-extras": {

  1215. 			// Method

  1216. 			{

  1217. 				`(?<operator>\.)(?<method_name>` + namePattern + `)` + colonPairLookahead + `\()`,

  1218. 				ByGroupNames(map[string]Emitter{

  1219. 					`operator`:    Operator,

  1220. 					`method_name`: NameFunction,

  1221. 				}),

  1222. 				Push(`name-adverb`),

  1223. 			},

  1224. 			// Function/Signature

  1225. 			{

  1226. 				`\(`, Punctuation, replaceRule(

  1227. 					ruleReplacingConfig{

  1228. 						delimiter: []rune(`)`),

  1229. 						tokenType: Punctuation,

  1230. 						stateName: `root`,

  1231. 						pushState: true,

  1232. 					}),

  1233. 			},

  1234. 			Default(Pop(1)),

  1235. 		},

  1236. 		"Q": {

  1237. 			Include("escape-qq"),

  1238. 			{`.+?`, String, nil},

  1239. 		},

  1240. 		"Q-closure": {

  1241. 			Include("escape-qq"),

  1242. 			Include("closure"),

  1243. 			{`.+?`, String, nil},

  1244. 		},

  1245. 		"Q-variable": {

  1246. 			Include("escape-qq"),

  1247. 			Include("qq-variable"),

  1248. 			{`.+?`, String, nil},

  1249. 		},

  1250. 		"closure": {

  1251. 			{`(?<!(?<!\\)\\){`, Punctuation, replaceRule(

  1252. 				ruleReplacingConfig{

  1253. 					delimiter: []rune(`}`),

  1254. 					tokenType: Punctuation,

  1255. 					stateName: `root`,

  1256. 					pushState: true,

  1257. 				}),

  1258. 			},

  1259. 		},

  1260. 		"token": {

  1261. 			// Token signature

  1262. 			{`\(`, Punctuation, replaceRule(

  1263. 				ruleReplacingConfig{

  1264. 					delimiter: []rune(`)`),

  1265. 					tokenType: Punctuation,

  1266. 					stateName: `root`,

  1267. 					pushState: true,

  1268. 				}),

  1269. 			},

  1270. 			{`{`, Punctuation, replaceRule(

  1271. 				ruleReplacingConfig{

  1272. 					delimiter: []rune(`}`),

  1273. 					tokenType: Punctuation,

  1274. 					stateName: `regex`,

  1275. 					popState:  true,

  1276. 					pushState: true,

  1277. 				}),

  1278. 			},

  1279. 			{`\s*`, Text, nil},

  1280. 			Default(Pop(1)),

  1281. 		},

  1282. 	}

  1283. }

  1284. 

  1285. // Joins keys of rune map

  1286. func joinRuneMap(m map[rune]rune) string {

  1287. 	runes := make([]rune, 0, len(m))

  1288. 	for k := range m {

  1289. 		runes = append(runes, k)

  1290. 	}

  1291. 

  1292. 	return string(runes)

  1293. }

  1294. 

  1295. // Finds the index of substring in the string starting at position n

  1296. func indexAt(str []rune, substr []rune, pos int) int {

  1297. 	strFromPos := str[pos:]

  1298. 	text := string(strFromPos)

  1299. 

  1300. 	idx := strings.Index(text, string(substr))

  1301. 	if idx > -1 {

  1302. 		idx = utf8.RuneCountInString(text[:idx])

  1303. 

  1304. 		// Search again if the substr is escaped with backslash

  1305. 		if (idx > 1 && strFromPos[idx-1] == '\\' && strFromPos[idx-2] != '\\') ||

  1306. 			(idx == 1 && strFromPos[idx-1] == '\\') {

  1307. 			idx = indexAt(str[pos:], substr, idx+1)

  1308. 

  1309. 			idx = utf8.RuneCountInString(text[:idx])

  1310. 

  1311. 			if idx < 0 {

  1312. 				return idx

  1313. 			}

  1314. 		}

  1315. 		idx += pos

  1316. 	}

  1317. 

  1318. 	return idx

  1319. }

  1320. 

  1321. // Tells if an array of string contains a string

  1322. func contains(s []string, e string) bool {

  1323. 	for _, value := range s {

  1324. 		if value == e {

  1325. 			return true

  1326. 		}

  1327. 	}

  1328. 	return false

  1329. }

  1330. 

  1331. type rulePosition int

  1332. 

  1333. const (

  1334. 	topRule    rulePosition = 0

  1335. 	bottomRule              = -1

  1336. )

  1337. 

  1338. type ruleMakingConfig struct {

  1339. 	delimiter              []rune

  1340. 	pattern                string

  1341. 	tokenType              Emitter

  1342. 	mutator                Mutator

  1343. 	numberOfDelimiterChars int

  1344. }

  1345. 

  1346. type ruleReplacingConfig struct {

  1347. 	delimiter              []rune

  1348. 	pattern                string

  1349. 	tokenType              Emitter

  1350. 	numberOfDelimiterChars int

  1351. 	mutator                Mutator

  1352. 	appendMutator          Mutator

  1353. 	rulePosition           rulePosition

  1354. 	stateName              string

  1355. 	pop                    bool

  1356. 	popState               bool

  1357. 	pushState              bool

  1358. }

  1359. 

  1360. // Pops rule from state-stack and replaces the rule with the previous rule

  1361. func popRule(rule ruleReplacingConfig) MutatorFunc {

  1362. 	return func(state *LexerState) error {

  1363. 		stackName := genStackName(rule.stateName, rule.rulePosition)

  1364. 

  1365. 		stack, ok := state.Get(stackName).([]ruleReplacingConfig)

  1366. 

  1367. 		if ok && len(stack) > 0 {

  1368. 			// Pop from stack

  1369. 			stack = stack[:len(stack)-1]

  1370. 			lastRule := stack[len(stack)-1]

  1371. 			lastRule.pushState = false

  1372. 			lastRule.popState = false

  1373. 			lastRule.pop = true

  1374. 			state.Set(stackName, stack)

  1375. 

  1376. 			// Call replaceRule to use the last rule

  1377. 			err := replaceRule(lastRule)(state)

  1378. 			if err != nil {

  1379. 				panic(err)

  1380. 			}

  1381. 		}

  1382. 

  1383. 		return nil

  1384. 	}

  1385. }

  1386. 

  1387. // Replaces a state's rule based on the rule config and position

  1388. func replaceRule(rule ruleReplacingConfig) MutatorFunc {

  1389. 	return func(state *LexerState) error {

  1390. 		stateName := rule.stateName

  1391. 		stackName := genStackName(rule.stateName, rule.rulePosition)

  1392. 

  1393. 		stack, ok := state.Get(stackName).([]ruleReplacingConfig)

  1394. 		if !ok {

  1395. 			stack = []ruleReplacingConfig{}

  1396. 		}

  1397. 

  1398. 		// If state-stack is empty fill it with the placeholder rule

  1399. 		if len(stack) == 0 {

  1400. 			stack = []ruleReplacingConfig{

  1401. 				{

  1402. 					// Placeholder, will be overwritten by mutators, DO NOT REMOVE!

  1403. 					pattern:      `\A\z`,

  1404. 					tokenType:    nil,

  1405. 					mutator:      nil,

  1406. 					stateName:    stateName,

  1407. 					rulePosition: rule.rulePosition,

  1408. 				},

  1409. 			}

  1410. 			state.Set(stackName, stack)

  1411. 		}

  1412. 

  1413. 		var mutator Mutator

  1414. 		mutators := []Mutator{}

  1415. 

  1416. 		switch {

  1417. 		case rule.rulePosition == topRule && rule.mutator == nil:

  1418. 			// Default mutator for top rule

  1419. 			mutators = []Mutator{Pop(1), popRule(rule)}

  1420. 		case rule.rulePosition == topRule && rule.mutator != nil:

  1421. 			// Default mutator for top rule, when rule.mutator is set

  1422. 			mutators = []Mutator{rule.mutator, popRule(rule)}

  1423. 		case rule.mutator != nil:

  1424. 			mutators = []Mutator{rule.mutator}

  1425. 		}

  1426. 

  1427. 		if rule.appendMutator != nil {

  1428. 			mutators = append(mutators, rule.appendMutator)

  1429. 		}

  1430. 

  1431. 		if len(mutators) > 0 {

  1432. 			mutator = Mutators(mutators...)

  1433. 		} else {

  1434. 			mutator = nil

  1435. 		}

  1436. 

  1437. 		ruleConfig := ruleMakingConfig{

  1438. 			pattern:                rule.pattern,

  1439. 			delimiter:              rule.delimiter,

  1440. 			numberOfDelimiterChars: rule.numberOfDelimiterChars,

  1441. 			tokenType:              rule.tokenType,

  1442. 			mutator:                mutator,

  1443. 		}

  1444. 

  1445. 		cRule := makeRule(ruleConfig)

  1446. 

  1447. 		switch rule.rulePosition {

  1448. 		case topRule:

  1449. 			state.Rules[stateName][0] = cRule

  1450. 		case bottomRule:

  1451. 			state.Rules[stateName][len(state.Rules[stateName])-1] = cRule

  1452. 		}

  1453. 

  1454. 		// Pop state name from stack if asked. State should be popped first before Pushing

  1455. 		if rule.popState {

  1456. 			err := Pop(1).Mutate(state)

  1457. 			if err != nil {

  1458. 				panic(err)

  1459. 			}

  1460. 		}

  1461. 

  1462. 		// Push state name to stack if asked

  1463. 		if rule.pushState {

  1464. 			err := Push(stateName).Mutate(state)

  1465. 			if err != nil {

  1466. 				panic(err)

  1467. 			}

  1468. 		}

  1469. 

  1470. 		if !rule.pop {

  1471. 			state.Set(stackName, append(stack, rule))

  1472. 		}

  1473. 

  1474. 		return nil

  1475. 	}

  1476. }

  1477. 

  1478. // Generates rule replacing stack using state name and rule position

  1479. func genStackName(stateName string, rulePosition rulePosition) (stackName string) {

  1480. 	switch rulePosition {

  1481. 	case topRule:

  1482. 		stackName = stateName + `-top-stack`

  1483. 	case bottomRule:

  1484. 		stackName = stateName + `-bottom-stack`

  1485. 	}

  1486. 	return

  1487. }

  1488. 

  1489. // Makes a compiled rule and returns it

  1490. func makeRule(config ruleMakingConfig) *CompiledRule {

  1491. 	var rePattern string

  1492. 

  1493. 	if len(config.delimiter) > 0 {

  1494. 		delimiter := string(config.delimiter)

  1495. 

  1496. 		if config.numberOfDelimiterChars > 1 {

  1497. 			delimiter = strings.Repeat(delimiter, config.numberOfDelimiterChars)

  1498. 		}

  1499. 

  1500. 		rePattern = `(?<!(?<!\\)\\)` + regexp2.Escape(delimiter)

  1501. 	} else {

  1502. 		rePattern = config.pattern

  1503. 	}

  1504. 

  1505. 	regex := regexp2.MustCompile(rePattern, regexp2.None)

  1506. 

  1507. 	cRule := &CompiledRule{

  1508. 		Rule:   Rule{rePattern, config.tokenType, config.mutator},

  1509. 		Regexp: regex,

  1510. 	}

  1511. 

  1512. 	return cRule

  1513. }

  1514. 

  1515. // Emitter for colon pairs, changes token state based on key and brackets

  1516. func colonPair(tokenClass TokenType) Emitter {

  1517. 	return EmitterFunc(func(groups []string, state *LexerState) Iterator {

  1518. 		iterators := []Iterator{}

  1519. 		tokens := []Token{

  1520. 			{Punctuation, state.NamedGroups[`colon`]},

  1521. 			{Punctuation, state.NamedGroups[`opening_delimiters`]},

  1522. 			{Punctuation, state.NamedGroups[`closing_delimiters`]},

  1523. 		}

  1524. 

  1525. 		// Append colon

  1526. 		iterators = append(iterators, Literator(tokens[0]))

  1527. 

  1528. 		if tokenClass == NameAttribute {

  1529. 			iterators = append(iterators, Literator(Token{NameAttribute, state.NamedGroups[`key`]}))

  1530. 		} else {

  1531. 			var keyTokenState string

  1532. 			keyre := regexp.MustCompile(`^\d+$`)

  1533. 			if keyre.MatchString(state.NamedGroups[`key`]) {

  1534. 				keyTokenState = "common"

  1535. 			} else {

  1536. 				keyTokenState = "Q"

  1537. 			}

  1538. 

  1539. 			// Use token state to Tokenise key

  1540. 			if keyTokenState != "" {

  1541. 				iterator, err := state.Lexer.Tokenise(

  1542. 					&TokeniseOptions{

  1543. 						State:  keyTokenState,

  1544. 						Nested: true,

  1545. 					}, state.NamedGroups[`key`])

  1546. 

  1547. 				if err != nil {

  1548. 					panic(err)

  1549. 				} else {

  1550. 					// Append key

  1551. 					iterators = append(iterators, iterator)

  1552. 				}

  1553. 			}

  1554. 		}

  1555. 

  1556. 		// Append punctuation

  1557. 		iterators = append(iterators, Literator(tokens[1]))

  1558. 

  1559. 		var valueTokenState string

  1560. 

  1561. 		switch state.NamedGroups[`opening_delimiters`] {

  1562. 		case "(", "{", "[":

  1563. 			valueTokenState = "root"

  1564. 		case "<<", "«":

  1565. 			valueTokenState = "ww"

  1566. 		case "<":

  1567. 			valueTokenState = "Q"

  1568. 		}

  1569. 

  1570. 		// Use token state to Tokenise value

  1571. 		if valueTokenState != "" {

  1572. 			iterator, err := state.Lexer.Tokenise(

  1573. 				&TokeniseOptions{

  1574. 					State:  valueTokenState,

  1575. 					Nested: true,

  1576. 				}, state.NamedGroups[`value`])

  1577. 

  1578. 			if err != nil {

  1579. 				panic(err)

  1580. 			} else {

  1581. 				// Append value

  1582. 				iterators = append(iterators, iterator)

  1583. 			}

  1584. 		}

  1585. 		// Append last punctuation

  1586. 		iterators = append(iterators, Literator(tokens[2]))

  1587. 

  1588. 		return Concaterator(iterators...)

  1589. 	})

  1590. }

  1591. 

  1592. // Emitter for quoting constructs, changes token state based on quote name and adverbs

  1593. func quote(groups []string, state *LexerState) Iterator {

  1594. 	keyword := state.NamedGroups[`keyword`]

  1595. 	adverbsStr := state.NamedGroups[`adverbs`]

  1596. 	iterators := []Iterator{}

  1597. 	tokens := []Token{

  1598. 		{Keyword, keyword},

  1599. 		{StringAffix, adverbsStr},

  1600. 		{Text, state.NamedGroups[`ws`]},

  1601. 		{Punctuation, state.NamedGroups[`opening_delimiters`]},

  1602. 		{Punctuation, state.NamedGroups[`closing_delimiters`]},

  1603. 	}

  1604. 

  1605. 	// Append all tokens before dealing with the main string

  1606. 	iterators = append(iterators, Literator(tokens[:4]...))

  1607. 

  1608. 	var tokenStates []string

  1609. 

  1610. 	// Set tokenStates based on adverbs

  1611. 	adverbs := strings.Split(adverbsStr, ":")

  1612. 	for _, adverb := range adverbs {

  1613. 		switch adverb {

  1614. 		case "c", "closure":

  1615. 			tokenStates = append(tokenStates, "Q-closure")

  1616. 		case "qq":

  1617. 			tokenStates = append(tokenStates, "qq")

  1618. 		case "ww":

  1619. 			tokenStates = append(tokenStates, "ww")

  1620. 		case "s", "scalar", "a", "array", "h", "hash", "f", "function":

  1621. 			tokenStates = append(tokenStates, "Q-variable")

  1622. 		}

  1623. 	}

  1624. 

  1625. 	var tokenState string

  1626. 

  1627. 	switch {

  1628. 	case keyword == "qq" || contains(tokenStates, "qq"):

  1629. 		tokenState = "qq"

  1630. 	case adverbsStr == "ww" || contains(tokenStates, "ww"):

  1631. 		tokenState = "ww"

  1632. 	case contains(tokenStates, "Q-closure") && contains(tokenStates, "Q-variable"):

  1633. 		tokenState = "qq"

  1634. 	case contains(tokenStates, "Q-closure"):

  1635. 		tokenState = "Q-closure"

  1636. 	case contains(tokenStates, "Q-variable"):

  1637. 		tokenState = "Q-variable"

  1638. 	default:

  1639. 		tokenState = "Q"

  1640. 	}

  1641. 

  1642. 	iterator, err := state.Lexer.Tokenise(

  1643. 		&TokeniseOptions{

  1644. 			State:  tokenState,

  1645. 			Nested: true,

  1646. 		}, state.NamedGroups[`value`])

  1647. 

  1648. 	if err != nil {

  1649. 		panic(err)

  1650. 	} else {

  1651. 		iterators = append(iterators, iterator)

  1652. 	}

  1653. 

  1654. 	// Append the last punctuation

  1655. 	iterators = append(iterators, Literator(tokens[4]))

  1656. 

  1657. 	return Concaterator(iterators...)

  1658. }

  1659. 

  1660. // Emitter for pod config, tokenises the properties with "colon-pair-attribute" state

  1661. func podConfig(groups []string, state *LexerState) Iterator {

  1662. 	// Tokenise pod config

  1663. 	iterator, err := state.Lexer.Tokenise(

  1664. 		&TokeniseOptions{

  1665. 			State:  "colon-pair-attribute",

  1666. 			Nested: true,

  1667. 		}, groups[0])

  1668. 

  1669. 	if err != nil {

  1670. 		panic(err)

  1671. 	} else {

  1672. 		return iterator

  1673. 	}

  1674. }

  1675. 

  1676. // Emitter for pod code, tokenises the code based on the lang specified

  1677. func podCode(groups []string, state *LexerState) Iterator {

  1678. 	iterators := []Iterator{}

  1679. 	tokens := []Token{

  1680. 		{Comment, state.NamedGroups[`ws`]},

  1681. 		{Keyword, state.NamedGroups[`keyword`]},

  1682. 		{Keyword, state.NamedGroups[`ws2`]},

  1683. 		{Keyword, state.NamedGroups[`name`]},

  1684. 		{StringDoc, state.NamedGroups[`value`]},

  1685. 		{Comment, state.NamedGroups[`ws3`]},

  1686. 		{Keyword, state.NamedGroups[`end_keyword`]},

  1687. 		{Keyword, state.NamedGroups[`ws4`]},

  1688. 		{Keyword, state.NamedGroups[`name`]},

  1689. 	}

  1690. 

  1691. 	// Append all tokens before dealing with the pod config

  1692. 	iterators = append(iterators, Literator(tokens[:4]...))

  1693. 

  1694. 	// Tokenise pod config

  1695. 	iterators = append(iterators, podConfig([]string{state.NamedGroups[`config`]}, state))

  1696. 

  1697. 	langMatch := regexp.MustCompile(`:lang\W+(\w+)`).FindStringSubmatch(state.NamedGroups[`config`])

  1698. 	var lang string

  1699. 	if len(langMatch) > 1 {

  1700. 		lang = langMatch[1]

  1701. 	}

  1702. 

  1703. 	// Tokenise code based on lang property

  1704. 	sublexer := Get(lang)

  1705. 	if sublexer != nil {

  1706. 		iterator, err := sublexer.Tokenise(nil, state.NamedGroups[`value`])

  1707. 

  1708. 		if err != nil {

  1709. 			panic(err)

  1710. 		} else {

  1711. 			iterators = append(iterators, iterator)

  1712. 		}

  1713. 	} else {

  1714. 		iterators = append(iterators, Literator(tokens[4]))

  1715. 	}

  1716. 

  1717. 	// Append the rest of the tokens

  1718. 	iterators = append(iterators, Literator(tokens[5:]...))

  1719. 

  1720. 	return Concaterator(iterators...)

  1721. }
About Sign Up Leave Feedback Contact