1// Take a look at the license at the top of the repository in the LICENSE file.
2
3use std::convert::TryFrom;
4use std::fmt;
5use std::str::{CharIndices, FromStr};
6
7#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
8pub enum ReservedChar {
9 Comma,
10 OpenParenthese,
11 CloseParenthese,
12 OpenCurlyBrace,
13 CloseCurlyBrace,
14 OpenBracket,
15 CloseBracket,
16 Colon,
17 SemiColon,
18 Dot,
19 Quote,
20 DoubleQuote,
21 ExclamationMark,
22 QuestionMark,
23 Slash,
24 Modulo,
25 Star,
26 Minus,
27 Plus,
28 EqualSign,
29 Backslash,
30 Space,
31 Tab,
32 Backline,
33 LessThan,
34 SuperiorThan,
35 Pipe,
36 Ampersand,
37 BackTick,
38}
39
40impl ReservedChar {
41 pub fn is_white_character(&self) -> bool {
42 *self == ReservedChar::Space
43 || *self == ReservedChar::Tab
44 || *self == ReservedChar::Backline
45 }
46}
47
48impl fmt::Display for ReservedChar {
49 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
50 write!(
51 f,
52 "{}",
53 match *self {
54 ReservedChar::Comma => ',',
55 ReservedChar::OpenParenthese => '(',
56 ReservedChar::CloseParenthese => ')',
57 ReservedChar::OpenCurlyBrace => '{',
58 ReservedChar::CloseCurlyBrace => '}',
59 ReservedChar::OpenBracket => '[',
60 ReservedChar::CloseBracket => ']',
61 ReservedChar::Colon => ':',
62 ReservedChar::SemiColon => ';',
63 ReservedChar::Dot => '.',
64 ReservedChar::Quote => '\'',
65 ReservedChar::DoubleQuote => '"',
66 ReservedChar::ExclamationMark => '!',
67 ReservedChar::QuestionMark => '?',
68 ReservedChar::Slash => '/',
69 ReservedChar::Modulo => '%',
70 ReservedChar::Star => '*',
71 ReservedChar::Minus => '-',
72 ReservedChar::Plus => '+',
73 ReservedChar::EqualSign => '=',
74 ReservedChar::Backslash => '\\',
75 ReservedChar::Space => ' ',
76 ReservedChar::Tab => '\t',
77 ReservedChar::Backline => '\n',
78 ReservedChar::LessThan => '<',
79 ReservedChar::SuperiorThan => '>',
80 ReservedChar::Pipe => '|',
81 ReservedChar::Ampersand => '&',
82 ReservedChar::BackTick => '`',
83 }
84 )
85 }
86}
87
88impl TryFrom<char> for ReservedChar {
89 type Error = &'static str;
90
91 fn try_from(value: char) -> Result<ReservedChar, Self::Error> {
92 match value {
93 ',' => Ok(ReservedChar::Comma),
94 '(' => Ok(ReservedChar::OpenParenthese),
95 ')' => Ok(ReservedChar::CloseParenthese),
96 '{' => Ok(ReservedChar::OpenCurlyBrace),
97 '}' => Ok(ReservedChar::CloseCurlyBrace),
98 '[' => Ok(ReservedChar::OpenBracket),
99 ']' => Ok(ReservedChar::CloseBracket),
100 ':' => Ok(ReservedChar::Colon),
101 ';' => Ok(ReservedChar::SemiColon),
102 '.' => Ok(ReservedChar::Dot),
103 '\'' => Ok(ReservedChar::Quote),
104 '"' => Ok(ReservedChar::DoubleQuote),
105 '!' => Ok(ReservedChar::ExclamationMark),
106 '?' => Ok(ReservedChar::QuestionMark),
107 '/' => Ok(ReservedChar::Slash),
108 '%' => Ok(ReservedChar::Modulo),
109 '*' => Ok(ReservedChar::Star),
110 '-' => Ok(ReservedChar::Minus),
111 '+' => Ok(ReservedChar::Plus),
112 '=' => Ok(ReservedChar::EqualSign),
113 '\\' => Ok(ReservedChar::Backslash),
114 ' ' => Ok(ReservedChar::Space),
115 '\t' => Ok(ReservedChar::Tab),
116 '\n' | '\r' => Ok(ReservedChar::Backline),
117 '<' => Ok(ReservedChar::LessThan),
118 '>' => Ok(ReservedChar::SuperiorThan),
119 '|' => Ok(ReservedChar::Pipe),
120 '&' => Ok(ReservedChar::Ampersand),
121 '`' => Ok(ReservedChar::BackTick),
122 _ => Err("Unknown reserved char"),
123 }
124 }
125}
126
127#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
128pub enum Keyword {
129 Break,
130 Case,
131 Catch,
132 Const,
133 Continue,
134 Default,
135 Do,
136 Else,
137 False,
138 Finally,
139 Function,
140 For,
141 If,
142 In,
143 InstanceOf,
144 Let,
145 New,
146 Null,
147 Private,
148 Protected,
149 Public,
150 Return,
151 Switch,
152 This,
153 Throw,
154 True,
155 Try,
156 Typeof,
157 Static,
158 Var,
159 While,
160}
161
162impl Keyword {
163 fn requires_before(&self) -> bool {
164 matches!(*self, Keyword::In | Keyword::InstanceOf)
165 }
166}
167
168impl fmt::Display for Keyword {
169 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
170 write!(
171 f,
172 "{}",
173 match *self {
174 Keyword::Break => "break",
175 Keyword::Case => "case",
176 Keyword::Catch => "catch",
177 Keyword::Const => "const",
178 Keyword::Continue => "continue",
179 Keyword::Default => "default",
180 Keyword::Do => "do",
181 Keyword::Else => "else",
182 Keyword::False => "false",
183 Keyword::Finally => "finally",
184 Keyword::Function => "function",
185 Keyword::For => "for",
186 Keyword::If => "if",
187 Keyword::In => "in",
188 Keyword::InstanceOf => "instanceof",
189 Keyword::Let => "let",
190 Keyword::New => "new",
191 Keyword::Null => "null",
192 Keyword::Private => "private",
193 Keyword::Protected => "protected",
194 Keyword::Public => "public",
195 Keyword::Return => "return",
196 Keyword::Switch => "switch",
197 Keyword::This => "this",
198 Keyword::Throw => "throw",
199 Keyword::True => "true",
200 Keyword::Try => "try",
201 Keyword::Typeof => "typeof",
202 Keyword::Static => "static",
203 Keyword::Var => "var",
204 Keyword::While => "while",
205 }
206 )
207 }
208}
209
210impl TryFrom<&str> for Keyword {
211 type Error = &'static str;
212
213 fn try_from(value: &str) -> Result<Keyword, Self::Error> {
214 match value {
215 "break" => Ok(Keyword::Break),
216 "case" => Ok(Keyword::Case),
217 "catch" => Ok(Keyword::Catch),
218 "const" => Ok(Keyword::Const),
219 "continue" => Ok(Keyword::Continue),
220 "default" => Ok(Keyword::Default),
221 "do" => Ok(Keyword::Do),
222 "else" => Ok(Keyword::Else),
223 "false" => Ok(Keyword::False),
224 "finally" => Ok(Keyword::Finally),
225 "function" => Ok(Keyword::Function),
226 "for" => Ok(Keyword::For),
227 "if" => Ok(Keyword::If),
228 "in" => Ok(Keyword::In),
229 "instanceof" => Ok(Keyword::InstanceOf),
230 "let" => Ok(Keyword::Let),
231 "new" => Ok(Keyword::New),
232 "null" => Ok(Keyword::Null),
233 "private" => Ok(Keyword::Private),
234 "protected" => Ok(Keyword::Protected),
235 "public" => Ok(Keyword::Public),
236 "return" => Ok(Keyword::Return),
237 "switch" => Ok(Keyword::Switch),
238 "this" => Ok(Keyword::This),
239 "throw" => Ok(Keyword::Throw),
240 "true" => Ok(Keyword::True),
241 "try" => Ok(Keyword::Try),
242 "typeof" => Ok(Keyword::Typeof),
243 "static" => Ok(Keyword::Static),
244 "var" => Ok(Keyword::Var),
245 "while" => Ok(Keyword::While),
246 _ => Err("Unkown keyword"),
247 }
248 }
249}
250
251#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
252pub enum Condition {
253 And,
254 Or,
255 DifferentThan,
256 SuperDifferentThan,
257 EqualTo,
258 SuperEqualTo,
259 SuperiorThan,
260 SuperiorOrEqualTo,
261 InferiorThan,
262 InferiorOrEqualTo,
263}
264
265impl fmt::Display for Condition {
266 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
267 write!(
268 f,
269 "{}",
270 match *self {
271 Condition::And => "&&",
272 Condition::Or => "||",
273 Condition::DifferentThan => "!=",
274 Condition::SuperDifferentThan => "!==",
275 Condition::EqualTo => "==",
276 Condition::SuperEqualTo => "===",
277 Condition::SuperiorThan => ">",
278 Condition::SuperiorOrEqualTo => ">=",
279 Condition::InferiorThan => "<",
280 Condition::InferiorOrEqualTo => "<=",
281 }
282 )
283 }
284}
285
286impl TryFrom<ReservedChar> for Condition {
287 type Error = &'static str;
288
289 fn try_from(value: ReservedChar) -> Result<Condition, Self::Error> {
290 Ok(match value {
291 ReservedChar::SuperiorThan => Condition::SuperiorThan,
292 ReservedChar::LessThan => Condition::InferiorThan,
293 _ => return Err("Unkown condition"),
294 })
295 }
296}
297
298#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
299pub enum Operation {
300 Addition,
301 AdditionEqual,
302 Subtract,
303 SubtractEqual,
304 Multiply,
305 MultiplyEqual,
306 Divide,
307 DivideEqual,
308 Modulo,
309 ModuloEqual,
310 Equal,
311}
312
313impl Operation {
314 pub fn is_assign(&self) -> bool {
315 matches!(
316 *self,
317 Operation::AdditionEqual
318 | Operation::SubtractEqual
319 | Operation::MultiplyEqual
320 | Operation::DivideEqual
321 | Operation::ModuloEqual
322 | Operation::Equal
323 )
324 }
325}
326
327impl fmt::Display for Operation {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 write!(
330 f,
331 "{}",
332 match *self {
333 Operation::Addition => "+",
334 Operation::AdditionEqual => "+=",
335 Operation::Subtract => "-",
336 Operation::SubtractEqual => "-=",
337 Operation::Multiply => "*",
338 Operation::MultiplyEqual => "*=",
339 Operation::Divide => "/",
340 Operation::DivideEqual => "/=",
341 Operation::Modulo => "%",
342 Operation::ModuloEqual => "%=",
343 Operation::Equal => "=",
344 }
345 )
346 }
347}
348
349impl TryFrom<ReservedChar> for Operation {
350 type Error = &'static str;
351
352 fn try_from(value: ReservedChar) -> Result<Operation, Self::Error> {
353 Ok(match value {
354 ReservedChar::Plus => Operation::Addition,
355 ReservedChar::Minus => Operation::Subtract,
356 ReservedChar::Slash => Operation::Divide,
357 ReservedChar::Star => Operation::Multiply,
358 ReservedChar::Modulo => Operation::Modulo,
359 ReservedChar::EqualSign => Operation::Equal,
360 _ => return Err("Unkown operation"),
361 })
362 }
363}
364
365#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Hash)]
366pub enum Token<'a> {
367 Keyword(Keyword),
368 Char(ReservedChar),
369 String(&'a str),
370 Comment(&'a str),
371 License(&'a str),
372 Other(&'a str),
373 Regex {
374 regex: &'a str,
375 is_global: bool,
376 is_interactive: bool,
377 },
378 Condition(Condition),
379 Operation(Operation),
380 CreatedVarDecl(String),
381 CreatedVar(String),
382 Number(usize),
383 FloatingNumber(&'a str),
384}
385
386impl fmt::Display for Token<'_> {
387 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
388 match *self {
389 Token::Keyword(x) => write!(f, "{}", x),
390 Token::Char(x) => write!(f, "{}", x),
391 Token::String(x) | Token::Comment(x) | Token::Other(x) => write!(f, "{}", x),
392 Token::License(x) => write!(f, "/*!{}*/", x),
393 Token::Regex {
394 regex,
395 is_global,
396 is_interactive,
397 } => {
398 let x = write!(f, "/{}/", regex);
399 if is_global {
400 write!(f, "g")?;
401 }
402 if is_interactive {
403 write!(f, "i")?;
404 }
405 x
406 }
407 Token::Condition(x) => write!(f, "{}", x),
408 Token::Operation(x) => write!(f, "{}", x),
409 Token::CreatedVarDecl(ref x) => write!(f, "{}", x),
410 Token::CreatedVar(ref x) => write!(f, "{}", x),
411 Token::Number(x) => write!(f, "{}", x),
412 Token::FloatingNumber(ref x) => write!(f, "{}", x),
413 }
414 }
415}
416
417impl Token<'_> {
418 pub fn is_comment(&self) -> bool {
419 matches!(*self, Token::Comment(_))
420 }
421
422 pub fn is_license(&self) -> bool {
423 matches!(*self, Token::License(_))
424 }
425
426 pub fn is_reserved_char(&self) -> bool {
427 matches!(*self, Token::Char(_))
428 }
429
430 pub fn get_char(&self) -> Option<ReservedChar> {
431 match *self {
432 Token::Char(c) => Some(c),
433 _ => None,
434 }
435 }
436
437 pub fn eq_char(&self, rc: ReservedChar) -> bool {
438 match *self {
439 Token::Char(c) => c == rc,
440 _ => false,
441 }
442 }
443
444 pub fn eq_operation(&self, ope: Operation) -> bool {
445 match *self {
446 Token::Operation(o) => o == ope,
447 _ => false,
448 }
449 }
450
451 pub fn is_operation(&self) -> bool {
452 matches!(*self, Token::Operation(_))
453 }
454
455 pub fn eq_condition(&self, cond: Condition) -> bool {
456 match *self {
457 Token::Condition(c) => c == cond,
458 _ => false,
459 }
460 }
461
462 pub fn is_condition(&self) -> bool {
463 matches!(*self, Token::Condition(_))
464 }
465
466 pub fn is_other(&self) -> bool {
467 matches!(*self, Token::Other(_))
468 }
469
470 pub fn get_other(&self) -> Option<&str> {
471 match *self {
472 Token::Other(s) => Some(s),
473 _ => None,
474 }
475 }
476
477 pub fn is_white_character(&self) -> bool {
478 match *self {
479 Token::Char(c) => c.is_white_character(),
480 _ => false,
481 }
482 }
483
484 pub fn is_keyword(&self) -> bool {
485 matches!(*self, Token::Keyword(_))
486 }
487
488 pub fn get_keyword(&self) -> Option<Keyword> {
489 match *self {
490 Token::Keyword(k) => Some(k),
491 _ => None,
492 }
493 }
494
495 pub fn is_string(&self) -> bool {
496 matches!(*self, Token::String(_))
497 }
498
499 pub fn get_string(&self) -> Option<&str> {
500 match *self {
501 Token::String(s) => Some(s),
502 _ => None,
503 }
504 }
505
506 pub fn is_regex(&self) -> bool {
507 matches!(*self, Token::Regex { .. })
508 }
509
510 pub fn is_created_var_decl(&self) -> bool {
511 matches!(*self, Token::CreatedVarDecl(_))
512 }
513
514 pub fn is_created_var(&self) -> bool {
515 matches!(*self, Token::CreatedVar(_))
516 }
517
518 pub fn is_number(&self) -> bool {
519 matches!(*self, Token::Number(_))
520 }
521
522 pub fn is_floating_number(&self) -> bool {
523 matches!(*self, Token::FloatingNumber(_))
524 }
525
526 fn get_required(&self) -> Option<char> {
527 match *self {
528 Token::Keyword(_)
529 | Token::Other(_)
530 | Token::CreatedVarDecl(_)
531 | Token::Number(_)
532 | Token::FloatingNumber(_) => Some(' '),
533 _ => None,
534 }
535 }
536
537 fn requires_before(&self) -> bool {
538 match *self {
539 Token::Keyword(k) => k.requires_before(),
540 _ => false,
541 }
542 }
543}
544
545fn get_line_comment<'a>(
546 source: &'a str,
547 iterator: &mut MyPeekable<'_>,
548 start_pos: &mut usize,
549) -> Option<Token<'a>> {
550 *start_pos += 1;
551 for (pos: usize, c: char) in iterator {
552 if let Ok(c: ReservedChar) = ReservedChar::try_from(c) {
553 if c == ReservedChar::Backline {
554 let ret: Option> = Some(Token::Comment(&source[*start_pos..pos]));
555 *start_pos = pos;
556 return ret;
557 }
558 }
559 }
560 None
561}
562
563fn get_regex<'a>(
564 source: &'a str,
565 iterator: &mut MyPeekable<'_>,
566 start_pos: &mut usize,
567 v: &[Token<'_>],
568) -> Option<Token<'a>> {
569 let mut back = v.len();
570 while back > 0 {
571 back -= 1;
572 if v[back].is_white_character() || v[back].is_comment() || v[back].is_license() {
573 continue;
574 }
575 match &v[back] {
576 Token::Char(ReservedChar::SemiColon)
577 | Token::Char(ReservedChar::Colon)
578 | Token::Char(ReservedChar::Comma)
579 | Token::Char(ReservedChar::OpenBracket)
580 | Token::Char(ReservedChar::OpenParenthese)
581 | Token::Char(ReservedChar::ExclamationMark)
582 | Token::Char(ReservedChar::OpenCurlyBrace)
583 | Token::Char(ReservedChar::QuestionMark)
584 | Token::Char(ReservedChar::Backline)
585 | Token::Char(ReservedChar::Pipe)
586 | Token::Char(ReservedChar::Ampersand) => break,
587 t if t.is_operation() || t.is_condition() => break,
588 _ => return None,
589 }
590 }
591 iterator.start_save();
592 while let Some((pos, c)) = iterator.next() {
593 if c == '\\' {
594 // we skip next character
595 iterator.next();
596 continue;
597 }
598 if let Ok(c) = ReservedChar::try_from(c) {
599 if c == ReservedChar::Slash {
600 let mut is_global = false;
601 let mut is_interactive = false;
602 let mut add = 0;
603 loop {
604 match iterator.peek() {
605 Some((_, 'i')) => is_interactive = true,
606 Some((_, 'g')) => is_global = true,
607 _ => break,
608 };
609 iterator.next();
610 add += 1;
611 }
612 let ret = Some(Token::Regex {
613 regex: &source[*start_pos + 1..pos],
614 is_interactive,
615 is_global,
616 });
617 *start_pos = pos + add;
618 iterator.drop_save();
619 return ret;
620 } else if c == ReservedChar::Backline {
621 break;
622 }
623 }
624 }
625 iterator.stop_save();
626 None
627}
628
629fn get_comment<'a>(
630 source: &'a str,
631 iterator: &mut MyPeekable<'_>,
632 start_pos: &mut usize,
633) -> Token<'a> {
634 let mut prev = ReservedChar::Quote;
635 *start_pos += 1;
636 let builder = if let Some((_, c)) = iterator.next() {
637 if c == '!' {
638 *start_pos += 1;
639 Token::License
640 } else {
641 if let Ok(c) = ReservedChar::try_from(c) {
642 prev = c;
643 }
644 Token::Comment
645 }
646 } else {
647 Token::Comment
648 };
649
650 let mut current_pos = *start_pos;
651 for (pos, c) in iterator {
652 current_pos = pos;
653 if let Ok(c) = ReservedChar::try_from(c) {
654 if c == ReservedChar::Slash && prev == ReservedChar::Star {
655 current_pos -= 2;
656 break;
657 }
658 prev = c;
659 } else {
660 prev = ReservedChar::Space;
661 }
662 }
663 // Unclosed comment so returning it anyway...
664 let ret = builder(&source[*start_pos..=current_pos]);
665 *start_pos = current_pos + 2;
666 ret
667}
668
669fn get_string<'a>(
670 source: &'a str,
671 iterator: &mut MyPeekable<'_>,
672 start_pos: &mut usize,
673 start: ReservedChar,
674) -> Option<Token<'a>> {
675 while let Some((pos: usize, c: char)) = iterator.next() {
676 if c == '\\' {
677 // we skip next character
678 iterator.next();
679 continue;
680 }
681 if let Ok(c: ReservedChar) = ReservedChar::try_from(c) {
682 if c == start {
683 let ret: Option> = Some(Token::String(&source[*start_pos..pos + 1]));
684 *start_pos = pos;
685 return ret;
686 }
687 }
688 }
689 None
690}
691
692fn get_backtick_string<'a>(
693 source: &'a str,
694 iterator: &mut MyPeekable<'_>,
695 start_pos: &mut usize,
696) -> Option<Token<'a>> {
697 while let Some((pos, c)) = iterator.next() {
698 if c == '\\' {
699 // we skip next character
700 iterator.next();
701 continue;
702 }
703 if c == '$' && iterator.peek().map(|(_, c)| c == '{').unwrap_or(false) {
704 let mut count = 0;
705
706 loop {
707 if let Some((mut pos, c)) = iterator.next() {
708 if c == '\\' {
709 // we skip next character
710 iterator.next();
711 continue;
712 } else if c == '"' || c == '\'' {
713 // We don't care about the result
714 get_string(
715 source,
716 iterator,
717 &mut pos,
718 ReservedChar::try_from(c)
719 .expect("ReservedChar::try_from unexpectedly failed..."),
720 );
721 } else if c == '`' {
722 get_backtick_string(source, iterator, &mut pos);
723 } else if c == '{' {
724 count += 1;
725 } else if c == '}' {
726 count -= 1;
727 if count == 0 {
728 break;
729 }
730 }
731 } else {
732 return None;
733 }
734 }
735 } else if c == '`' {
736 let ret = Some(Token::String(&source[*start_pos..pos + 1]));
737 *start_pos = pos;
738 return ret;
739 }
740 }
741 None
742}
743
744fn first_useful<'a>(v: &'a [Token<'a>]) -> Option<&'a Token<'a>> {
745 for x: &Token<'_> in v.iter().rev() {
746 if x.is_white_character() {
747 continue;
748 }
749 return Some(x);
750 }
751 None
752}
753
754fn fill_other<'a>(source: &'a str, v: &mut Vec<Token<'a>>, start: usize, pos: usize) {
755 if start < pos {
756 if let Ok(w: Keyword) = Keyword::try_from(&source[start..pos]) {
757 v.push(Token::Keyword(w));
758 } else if let Ok(n: usize) = usize::from_str(&source[start..pos]) {
759 v.push(Token::Number(n))
760 } else if f64::from_str(&source[start..pos]).is_ok() {
761 v.push(Token::FloatingNumber(&source[start..pos]))
762 } else {
763 v.push(Token::Other(&source[start..pos]));
764 }
765 }
766}
767
768fn handle_equal_sign(v: &mut Vec<Token<'_>>, c: ReservedChar) -> bool {
769 if c != ReservedChar::EqualSign {
770 return false;
771 }
772 match v.last().unwrap_or(&Token::Other("")) {
773 Token::Operation(Operation::Equal) => {
774 v.pop();
775 v.push(Token::Condition(Condition::EqualTo));
776 }
777 Token::Condition(Condition::EqualTo) => {
778 v.pop();
779 v.push(Token::Condition(Condition::SuperEqualTo));
780 }
781 Token::Char(ReservedChar::ExclamationMark) => {
782 v.pop();
783 v.push(Token::Condition(Condition::DifferentThan));
784 }
785 Token::Condition(Condition::DifferentThan) => {
786 v.pop();
787 v.push(Token::Condition(Condition::SuperDifferentThan));
788 }
789 Token::Operation(Operation::Divide) => {
790 v.pop();
791 v.push(Token::Operation(Operation::DivideEqual));
792 }
793 Token::Operation(Operation::Multiply) => {
794 v.pop();
795 v.push(Token::Operation(Operation::MultiplyEqual));
796 }
797 Token::Operation(Operation::Addition) => {
798 v.pop();
799 v.push(Token::Operation(Operation::AdditionEqual));
800 }
801 Token::Operation(Operation::Subtract) => {
802 v.pop();
803 v.push(Token::Operation(Operation::SubtractEqual));
804 }
805 Token::Operation(Operation::Modulo) => {
806 v.pop();
807 v.push(Token::Operation(Operation::ModuloEqual));
808 }
809 Token::Condition(Condition::SuperiorThan) => {
810 v.pop();
811 v.push(Token::Condition(Condition::SuperiorOrEqualTo));
812 }
813 Token::Condition(Condition::InferiorThan) => {
814 v.pop();
815 v.push(Token::Condition(Condition::InferiorOrEqualTo));
816 }
817 _ => {
818 return false;
819 }
820 }
821 true
822}
823
824fn check_if_number(iterator: &mut MyPeekable<'_>, start: usize, pos: usize, source: &str) -> bool {
825 if source[start..pos].find('.').is_some() {
826 return false;
827 } else if u64::from_str(&source[start..pos]).is_ok() {
828 return true;
829 } else if let Some((_, x: char)) = iterator.peek() {
830 return x as u8 >= b'0' && x as u8 <= b'9';
831 }
832 false
833}
834
835struct MyPeekable<'a> {
836 inner: CharIndices<'a>,
837 saved: Vec<(usize, char)>,
838 peeked: Option<(usize, char)>,
839 is_saving: bool,
840}
841
842impl<'a> MyPeekable<'a> {
843 fn new(indices: CharIndices<'a>) -> MyPeekable<'a> {
844 MyPeekable {
845 inner: indices,
846 saved: Vec::with_capacity(500),
847 peeked: None,
848 is_saving: false,
849 }
850 }
851
852 fn start_save(&mut self) {
853 self.is_saving = true;
854 if let Some(p) = self.peeked {
855 self.saved.push(p);
856 }
857 }
858
859 fn drop_save(&mut self) {
860 self.is_saving = false;
861 self.saved.clear();
862 }
863
864 fn stop_save(&mut self) {
865 self.is_saving = false;
866 if let Some(p) = self.peeked {
867 self.saved.push(p);
868 }
869 self.peeked = None;
870 }
871
872 /// Returns None if saving.
873 fn peek(&mut self) -> Option<(usize, char)> {
874 if self.peeked.is_none() {
875 self.peeked = self.inner.next();
876 if self.is_saving {
877 if let Some(p) = self.peeked {
878 self.saved.push(p);
879 }
880 }
881 }
882 self.peeked
883 }
884}
885
886impl Iterator for MyPeekable<'_> {
887 type Item = (usize, char);
888
889 fn next(&mut self) -> Option<Self::Item> {
890 if self.peeked.is_some() {
891 self.peeked.take()
892 } else {
893 if !self.is_saving && !self.saved.is_empty() {
894 return Some(self.saved.remove(index:0));
895 }
896 match self.inner.next() {
897 Some(r: (usize, char)) if self.is_saving => {
898 self.saved.push(r);
899 Some(r)
900 }
901 r: Option<(usize, char)> => r,
902 }
903 }
904 }
905}
906
907pub fn tokenize(source: &str) -> Tokens<'_> {
908 let mut v = Vec::with_capacity(1000);
909 let mut start = 0;
910 let mut iterator = MyPeekable::new(source.char_indices());
911
912 loop {
913 let (mut pos, c) = match iterator.next() {
914 Some(x) => x,
915 None => {
916 fill_other(source, &mut v, start, source.len());
917 break;
918 }
919 };
920 if let Ok(c) = ReservedChar::try_from(c) {
921 if c == ReservedChar::Dot && check_if_number(&mut iterator, start, pos, source) {
922 let mut cont = true;
923 if let Some(x) = iterator.peek() {
924 if !"0123456789,; \t\n<>/*&|{}[]-+=~%^:!".contains(x.1) {
925 fill_other(source, &mut v, start, pos);
926 start = pos;
927 cont = false;
928 }
929 }
930 if cont {
931 continue;
932 }
933 }
934 fill_other(source, &mut v, start, pos);
935 match c {
936 ReservedChar::Quote | ReservedChar::DoubleQuote => {
937 if let Some(s) = get_string(source, &mut iterator, &mut pos, c) {
938 v.push(s);
939 }
940 }
941 ReservedChar::BackTick => {
942 if let Some(s) = get_backtick_string(source, &mut iterator, &mut pos) {
943 v.push(s);
944 }
945 }
946 ReservedChar::Slash
947 if v.last()
948 .unwrap_or(&Token::Other(""))
949 .eq_operation(Operation::Divide) =>
950 {
951 v.pop();
952 if let Some(s) = get_line_comment(source, &mut iterator, &mut pos) {
953 v.push(s);
954 }
955 }
956 ReservedChar::Slash
957 if iterator.peek().is_some()
958 && iterator.peek().unwrap().1 != '/'
959 && iterator.peek().unwrap().1 != '*'
960 && !first_useful(&v).unwrap_or(&Token::String("")).is_other() =>
961 {
962 if let Some(r) = get_regex(source, &mut iterator, &mut pos, &v) {
963 v.push(r);
964 } else {
965 v.push(Token::Operation(Operation::Divide));
966 }
967 }
968 ReservedChar::Star
969 if v.last()
970 .unwrap_or(&Token::Other(""))
971 .eq_operation(Operation::Divide) =>
972 {
973 v.pop();
974 v.push(get_comment(source, &mut iterator, &mut pos));
975 }
976 ReservedChar::Pipe
977 if v.last()
978 .unwrap_or(&Token::Other(""))
979 .eq_char(ReservedChar::Pipe) =>
980 {
981 v.pop();
982 v.push(Token::Condition(Condition::Or));
983 }
984 ReservedChar::Ampersand
985 if v.last()
986 .unwrap_or(&Token::Other(""))
987 .eq_char(ReservedChar::Ampersand) =>
988 {
989 v.pop();
990 v.push(Token::Condition(Condition::And));
991 }
992 _ if handle_equal_sign(&mut v, c) => {}
993 _ => {
994 if let Ok(o) = Operation::try_from(c) {
995 v.push(Token::Operation(o));
996 } else if let Ok(o) = Condition::try_from(c) {
997 v.push(Token::Condition(o));
998 } else {
999 v.push(Token::Char(c));
1000 }
1001 }
1002 }
1003 start = pos + 1;
1004 }
1005 }
1006 Tokens(v)
1007}
1008
1009#[derive(Debug, PartialEq, Eq)]
1010pub struct Tokens<'a>(pub Vec<Token<'a>>);
1011
1012macro_rules! tokens_writer {
1013 ($self:ident, $w:ident) => {
1014 let tokens = &$self.0;
1015 for i in 0..tokens.len() {
1016 if i > 0
1017 && tokens[i].requires_before()
1018 && !tokens[i - 1].is_keyword()
1019 && !tokens[i - 1].is_other()
1020 && !tokens[i - 1].is_reserved_char()
1021 && !tokens[i - 1].is_string()
1022 {
1023 write!($w, " ")?;
1024 }
1025 write!($w, "{}", tokens[i])?;
1026 if let Some(c) = match tokens[i] {
1027 Token::Keyword(_) | Token::Other(_) if i + 1 < tokens.len() => {
1028 tokens[i + 1].get_required()
1029 }
1030 _ => None,
1031 } {
1032 write!($w, "{}", c)?;
1033 }
1034 }
1035 };
1036}
1037
1038impl Tokens<'_> {
1039 pub(super) fn write<W: std::io::Write>(self, mut w: W) -> std::io::Result<()> {
1040 tokens_writer!(self, w);
1041 Ok(())
1042 }
1043}
1044
1045impl fmt::Display for Tokens<'_> {
1046 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1047 tokens_writer!(self, f);
1048 Ok(())
1049 }
1050}
1051
1052impl<'a> Tokens<'a> {
1053 #[must_use]
1054 pub fn apply<F>(self, func: F) -> Tokens<'a>
1055 where
1056 F: Fn(Tokens<'a>) -> Tokens<'a>,
1057 {
1058 func(self)
1059 }
1060}
1061
1062impl<'a> IntoIterator for Tokens<'a> {
1063 type Item = Token<'a>;
1064 type IntoIter = std::vec::IntoIter<Token<'a>>;
1065
1066 fn into_iter(self) -> Self::IntoIter {
1067 self.0.into_iter()
1068 }
1069}
1070
1071impl<'a> std::ops::Deref for Tokens<'a> {
1072 type Target = Vec<Token<'a>>;
1073
1074 fn deref(&self) -> &Self::Target {
1075 &self.0
1076 }
1077}
1078
1079impl<'a> From<Vec<Token<'a>>> for Tokens<'a> {
1080 fn from(v: Vec<Token<'a>>) -> Self {
1081 Tokens(v)
1082 }
1083}
1084
1085impl<'a> From<&[Token<'a>]> for Tokens<'a> {
1086 fn from(v: &[Token<'a>]) -> Self {
1087 Tokens(v.to_vec())
1088 }
1089}
1090
1091#[test]
1092fn check_regex() {
1093 let source = r#"var x = /"\.x/g;"#;
1094 let expected_result = r#"var x=/"\.x/g"#;
1095 assert_eq!(crate::js::minify(source).to_string(), expected_result);
1096
1097 let v = tokenize(source).apply(crate::js::clean_tokens);
1098 assert_eq!(
1099 v.0[3],
1100 Token::Regex {
1101 regex: "\"\\.x",
1102 is_global: true,
1103 is_interactive: false,
1104 }
1105 );
1106
1107 let source = r#"var x = /"\.x/gigigigig;var x = "hello";"#;
1108 let expected_result = r#"var x=/"\.x/gi;var x="hello""#;
1109 assert_eq!(crate::js::minify(source).to_string(), expected_result);
1110
1111 let v = tokenize(source).apply(crate::js::clean_tokens);
1112 assert_eq!(
1113 v.0[3],
1114 Token::Regex {
1115 regex: "\"\\.x",
1116 is_global: true,
1117 is_interactive: true,
1118 }
1119 );
1120}
1121
1122#[test]
1123fn more_regex() {
1124 let source = r#"var x = /"\.x\/a/i;"#;
1125 let expected_result = r#"var x=/"\.x\/a/i"#;
1126 assert_eq!(crate::js::minify(source).to_string(), expected_result);
1127
1128 let v = tokenize(source).apply(crate::js::clean_tokens);
1129 assert_eq!(
1130 v.0[3],
1131 Token::Regex {
1132 regex: "\"\\.x\\/a",
1133 is_global: false,
1134 is_interactive: true,
1135 }
1136 );
1137
1138 let source = r#"var x = /\\/i;"#;
1139 let expected_result = r#"var x=/\\/i"#;
1140 assert_eq!(crate::js::minify(source).to_string(), expected_result);
1141
1142 let v = tokenize(source).apply(crate::js::clean_tokens);
1143 assert_eq!(
1144 v.0[3],
1145 Token::Regex {
1146 regex: "\\\\",
1147 is_global: false,
1148 is_interactive: true,
1149 }
1150 );
1151}
1152
1153#[test]
1154fn even_more_regex() {
1155 let source = r#"var x = /a-z /;"#;
1156
1157 let v = tokenize(source).apply(crate::js::clean_tokens);
1158 assert_eq!(
1159 v.0[3],
1160 Token::Regex {
1161 regex: "a-z ",
1162 is_global: false,
1163 is_interactive: false,
1164 }
1165 );
1166}
1167
1168#[test]
1169fn not_regex_test() {
1170 let source = "( x ) / 2; x / y;x /= y";
1171
1172 let v = tokenize(source).apply(crate::js::clean_tokens);
1173 assert_eq!(
1174 &v.0,
1175 &[
1176 Token::Char(ReservedChar::OpenParenthese),
1177 Token::Other("x"),
1178 Token::Char(ReservedChar::CloseParenthese),
1179 Token::Operation(Operation::Divide),
1180 Token::Number(2),
1181 Token::Char(ReservedChar::SemiColon),
1182 Token::Other("x"),
1183 Token::Operation(Operation::Divide),
1184 Token::Other("y"),
1185 Token::Char(ReservedChar::SemiColon),
1186 Token::Other("x"),
1187 Token::Operation(Operation::DivideEqual),
1188 Token::Other("y")
1189 ]
1190 );
1191
1192 let source = "let x = /x\ny/;";
1193
1194 let v = tokenize(source).apply(crate::js::clean_tokens);
1195 assert_eq!(
1196 &v.0,
1197 &[
1198 Token::Keyword(Keyword::Let),
1199 Token::Other("x"),
1200 Token::Operation(Operation::Equal),
1201 Token::Operation(Operation::Divide),
1202 Token::Other("x"),
1203 Token::Other("y"),
1204 Token::Operation(Operation::Divide)
1205 ]
1206 );
1207}
1208
1209#[test]
1210fn test_tokens_parsing() {
1211 let source = "true = == 2.3 === 32";
1212
1213 let v = tokenize(source).apply(crate::js::clean_tokens);
1214 assert_eq!(
1215 &v.0,
1216 &[
1217 Token::Keyword(Keyword::True),
1218 Token::Operation(Operation::Equal),
1219 Token::Condition(Condition::EqualTo),
1220 Token::FloatingNumber("2.3"),
1221 Token::Condition(Condition::SuperEqualTo),
1222 Token::Number(32)
1223 ]
1224 );
1225}
1226
1227#[test]
1228fn test_string_parsing() {
1229 let source = "var x = 'hello people!'";
1230
1231 let v = tokenize(source).apply(crate::js::clean_tokens);
1232 assert_eq!(
1233 &v.0,
1234 &[
1235 Token::Keyword(Keyword::Var),
1236 Token::Other("x"),
1237 Token::Operation(Operation::Equal),
1238 Token::String("\'hello people!\'")
1239 ]
1240 );
1241}
1242
1243#[test]
1244fn test_number_parsing() {
1245 let source = "var x = .12; let y = 4.; var z = 12; .3 4. 'a' let u = 12.2";
1246
1247 let v = tokenize(source).apply(crate::js::clean_tokens);
1248 assert_eq!(
1249 &v.0,
1250 &[
1251 Token::Keyword(Keyword::Var),
1252 Token::Other("x"),
1253 Token::Operation(Operation::Equal),
1254 Token::FloatingNumber(".12"),
1255 Token::Char(ReservedChar::SemiColon),
1256 Token::Keyword(Keyword::Let),
1257 Token::Other("y"),
1258 Token::Operation(Operation::Equal),
1259 Token::FloatingNumber("4."),
1260 Token::Char(ReservedChar::SemiColon),
1261 Token::Keyword(Keyword::Var),
1262 Token::Other("z"),
1263 Token::Operation(Operation::Equal),
1264 Token::Number(12),
1265 Token::Char(ReservedChar::SemiColon),
1266 Token::FloatingNumber(".3"),
1267 Token::FloatingNumber("4."),
1268 Token::String("'a'"),
1269 Token::Keyword(Keyword::Let),
1270 Token::Other("u"),
1271 Token::Operation(Operation::Equal),
1272 Token::FloatingNumber("12.2")
1273 ]
1274 );
1275}
1276
1277#[test]
1278fn test_number_parsing2() {
1279 let source = "var x = 12.a;";
1280
1281 let v = tokenize(source).apply(crate::js::clean_tokens);
1282 assert_eq!(
1283 &v.0,
1284 &[
1285 Token::Keyword(Keyword::Var),
1286 Token::Other("x"),
1287 Token::Operation(Operation::Equal),
1288 Token::Number(12),
1289 Token::Char(ReservedChar::Dot),
1290 Token::Other("a")
1291 ]
1292 );
1293}
1294
1295#[test]
1296fn tokens_spaces() {
1297 let source = "t in e";
1298
1299 let v = tokenize(source).apply(crate::js::clean_tokens);
1300 assert_eq!(
1301 &v.0,
1302 &[
1303 Token::Other("t"),
1304 Token::Keyword(Keyword::In),
1305 Token::Other("e")
1306 ]
1307 );
1308}
1309
1310#[test]
1311fn division_by_id() {
1312 let source = "100/abc";
1313
1314 let v = tokenize(source).apply(crate::js::clean_tokens);
1315 assert_eq!(
1316 &v.0,
1317 &[
1318 Token::Number(100),
1319 Token::Operation(Operation::Divide),
1320 Token::Other("abc")
1321 ]
1322 );
1323}
1324
1325#[test]
1326fn weird_regex() {
1327 let source = "if (!/\\/(contact|legal)\\//.test(a)) {}";
1328
1329 let v = tokenize(source).apply(crate::js::clean_tokens);
1330 assert_eq!(
1331 &v.0,
1332 &[
1333 Token::Keyword(Keyword::If),
1334 Token::Char(ReservedChar::OpenParenthese),
1335 Token::Char(ReservedChar::ExclamationMark),
1336 Token::Regex {
1337 regex: "\\/(contact|legal)\\/",
1338 is_global: false,
1339 is_interactive: false
1340 },
1341 Token::Char(ReservedChar::Dot),
1342 Token::Other("test"),
1343 Token::Char(ReservedChar::OpenParenthese),
1344 Token::Other("a"),
1345 Token::Char(ReservedChar::CloseParenthese),
1346 Token::Char(ReservedChar::CloseParenthese),
1347 Token::Char(ReservedChar::OpenCurlyBrace),
1348 Token::Char(ReservedChar::CloseCurlyBrace),
1349 ]
1350 );
1351}
1352
1353#[test]
1354fn test_regexes() {
1355 let source = "/\\/(contact|legal)\\//.test";
1356
1357 let v = tokenize(source).apply(crate::js::clean_tokens);
1358 assert_eq!(
1359 &v.0,
1360 &[
1361 Token::Regex {
1362 regex: "\\/(contact|legal)\\/",
1363 is_global: false,
1364 is_interactive: false
1365 },
1366 Token::Char(ReservedChar::Dot),
1367 Token::Other("test"),
1368 ]
1369 );
1370
1371 let source = "/\\*(contact|legal)/.test";
1372
1373 let v = tokenize(source).apply(crate::js::clean_tokens);
1374 assert_eq!(
1375 &v.0,
1376 &[
1377 Token::Regex {
1378 regex: "\\*(contact|legal)",
1379 is_global: false,
1380 is_interactive: false
1381 },
1382 Token::Char(ReservedChar::Dot),
1383 Token::Other("test"),
1384 ]
1385 );
1386}
1387
1388#[test]
1389fn test_comments() {
1390 let source = "/*(contact|legal)/.test";
1391
1392 let v = tokenize(source);
1393 assert_eq!(&v.0, &[Token::Comment("(contact|legal)/.test"),],);
1394
1395 let source = "/*(contact|legal)/.test*/ a";
1396
1397 let v = tokenize(source);
1398 assert_eq!(
1399 &v.0,
1400 &[
1401 Token::Comment("(contact|legal)/.test"),
1402 Token::Char(ReservedChar::Space),
1403 Token::Other("a"),
1404 ],
1405 );
1406}
1407