1. package chroma
    
  2. 
    
  3. import "strings"
    
  4. 
    
  5. // An Iterator across tokens.
    
  6. //
    
  7. // EOF will be returned at the end of the Token stream.
    
  8. //
    
  9. // If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
    
  10. type Iterator func() Token
    
  11. 
    
  12. // Tokens consumes all tokens from the iterator and returns them as a slice.
    
  13. func (i Iterator) Tokens() []Token {
    
  14. 	var out []Token
    
  15. 	for t := i(); t != EOF; t = i() {
    
  16. 		out = append(out, t)
    
  17. 	}
    
  18. 	return out
    
  19. }
    
  20. 
    
  21. // Concaterator concatenates tokens from a series of iterators.
    
  22. func Concaterator(iterators ...Iterator) Iterator {
    
  23. 	return func() Token {
    
  24. 		for len(iterators) > 0 {
    
  25. 			t := iterators[0]()
    
  26. 			if t != EOF {
    
  27. 				return t
    
  28. 			}
    
  29. 			iterators = iterators[1:]
    
  30. 		}
    
  31. 		return EOF
    
  32. 	}
    
  33. }
    
  34. 
    
  35. // Literator converts a sequence of literal Tokens into an Iterator.
    
  36. func Literator(tokens ...Token) Iterator {
    
  37. 	return func() Token {
    
  38. 		if len(tokens) == 0 {
    
  39. 			return EOF
    
  40. 		}
    
  41. 		token := tokens[0]
    
  42. 		tokens = tokens[1:]
    
  43. 		return token
    
  44. 	}
    
  45. }
    
  46. 
    
  47. // SplitTokensIntoLines splits tokens containing newlines in two.
    
  48. func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
    
  49. 	var line []Token // nolint: prealloc
    
  50. 	for _, token := range tokens {
    
  51. 		for strings.Contains(token.Value, "\n") {
    
  52. 			parts := strings.SplitAfterN(token.Value, "\n", 2)
    
  53. 			// Token becomes the tail.
    
  54. 			token.Value = parts[1]
    
  55. 
    
  56. 			// Append the head to the line and flush the line.
    
  57. 			clone := token.Clone()
    
  58. 			clone.Value = parts[0]
    
  59. 			line = append(line, clone)
    
  60. 			out = append(out, line)
    
  61. 			line = nil
    
  62. 		}
    
  63. 		line = append(line, token)
    
  64. 	}
    
  65. 	if len(line) > 0 {
    
  66. 		out = append(out, line)
    
  67. 	}
    
  68. 	// Strip empty trailing token line.
    
  69. 	if len(out) > 0 {
    
  70. 		last := out[len(out)-1]
    
  71. 		if len(last) == 1 && last[0].Value == "" {
    
  72. 			out = out[:len(out)-1]
    
  73. 		}
    
  74. 	}
    
  75. 	return
    
  76. }