#![allow(non_camel_case_types)]

use std;
use std::str::FromStr;
use std::ascii::AsciiExt;
use std::collections::HashMap;

#[derive(Clone)]
#[derive(PartialEq)]
pub enum Token_Type {
  None,
  Function,
  And, Or, Xor, Not,
  Do, End, Return, Break, Goto,
  If, Then, Else, Else_If, For, While, Repeat, Until,
  Nil, False, True,
  Method_Call, Label_Marker, Member, Concat, Ellipsis,
  Equal, Greater, Greater_Equal, Lesser, Lesser_Equal, Not_Equal,
  Assign, Add, Subtract, Divide, Integer_Divide, Multiply, Exponent, Modulus, Length,
  Shift_Left, Shift_Right, Bitwise_And, Bitwise_Or, Bitwise_Not_Or_Xor,
  Int(i64), Float(f64), Str(String, char), Long_Str(String, i32),
  Local, Name(String),
  Element_Separator, Table_Open, Table_Close,
  Index_Open, Index_Close, Paren_Open, Paren_Close,
  Comment(String), Long_Comment(String, i32),
  Space(String), New_Line(String),
  Error(String),
}

#[derive(Clone)]
#[derive(PartialEq)]
pub struct Token {
  pub kind: Token_Type,
  pub line_id: usize,
  pub char_id: usize,
}

impl Token {
  pub fn new_blank() -> Token {
    Token{kind: Token_Type::None, line_id: 0, char_id: 0}
  }
  fn to_name(&self) -> String {
    match self.kind {
      Token_Type::None => "None".to_string(),
      Token_Type::Function => "Function".to_string(),
      Token_Type::And => "And".to_string(),
      Token_Type::Or => "Or".to_string(),
      Token_Type::Xor => "Xor".to_string(),
      Token_Type::Not => "Not".to_string(),
      Token_Type::Do => "Do".to_string(),
      Token_Type::End => "End".to_string(),
      Token_Type::Return => "Return".to_string(),
      Token_Type::Break => "Break".to_string(),
      Token_Type::Goto => "Goto".to_string(),
      Token_Type::If => "If".to_string(),
      Token_Type::Then => "Then".to_string(),
      Token_Type::Else => "Else".to_string(),
      Token_Type::Else_If => "Else_If".to_string(),
      Token_Type::For => "For".to_string(),
      Token_Type::While => "While".to_string(),
      Token_Type::Repeat => "Repeat".to_string(),
      Token_Type::Until => "Until".to_string(),
      Token_Type::Nil => "Nil".to_string(),
      Token_Type::False => "False".to_string(),
      Token_Type::True => "True".to_string(),
      Token_Type::Method_Call => "Method_Call".to_string(),
      Token_Type::Label_Marker => "Label_Marker".to_string(),
      Token_Type::Member => "Member".to_string(),
      Token_Type::Concat => "Concat".to_string(),
      Token_Type::Ellipsis => "Ellipsis".to_string(),
      Token_Type::Equal => "Equal".to_string(),
      Token_Type::Greater => "Greater".to_string(),
      Token_Type::Greater_Equal => "Greater_Equal".to_string(),
      Token_Type::Lesser => "Lesser".to_string(),
      Token_Type::Lesser_Equal => "Lesser_Equal".to_string(),
      Token_Type::Not_Equal => "Not_Equal".to_string(),
      Token_Type::Assign => "Assign".to_string(),
      Token_Type::Add => "Add".to_string(),
      Token_Type::Subtract => "Subtract".to_string(),
      Token_Type::Divide => "Divide".to_string(),
      Token_Type::Integer_Divide => "Integer_Divide".to_string(),
      Token_Type::Multiply => "Multiply".to_string(),
      Token_Type::Exponent => "Exponent".to_string(),
      Token_Type::Modulus => "Modulus".to_string(),
      Token_Type::Length => "Length".to_string(),
      Token_Type::Shift_Left => "Shift_Left".to_string(),
      Token_Type::Shift_Right => "Shift_Right".to_string(),
      Token_Type::Bitwise_And => "Bitwise_And".to_string(),
      Token_Type::Bitwise_Or => "Bitwise_Or".to_string(),
      Token_Type::Bitwise_Not_Or_Xor => "Bitwise_Not_Or_Xor".to_string(),
      Token_Type::Int(ref aval) => format!("Int : '{}'", aval),
      Token_Type::Float(ref aval) => format!("Float : '{}'", aval),
      Token_Type::Str(ref aval, ref bval) => format!("Str : '{}' '{}'", aval, bval),
      Token_Type::Long_Str(ref aval, ref bval) => format!("Long_Str : '{}' '{}'", aval, bval),
      Token_Type::Local => "Local".to_string(),
      Token_Type::Name(ref aval) => format!("Name : '{}'", aval),
      Token_Type::Element_Separator => "Element_Separator".to_string(),
      Token_Type::Table_Open => "Table_Open".to_string(),
      Token_Type::Table_Close => "Table_Close".to_string(),
      Token_Type::Index_Open => "Index_Open".to_string(),
      Token_Type::Index_Close => "Index_Close".to_string(),
      Token_Type::Paren_Open => "Paren_Open".to_string(),
      Token_Type::Paren_Close => "Paren_Close".to_string(),
      Token_Type::Comment(ref aval) => format!("Comment : '{}'", aval),
      Token_Type::Long_Comment(ref aval, ref bval) => format!("Long_Comment : '{}' '{}'", aval, bval),
      Token_Type::Space(ref aval) => format!("Space : '{}'", aval),
      Token_Type::New_Line(ref aval) => format!("New_Line : '{}'", aval),
      Token_Type::Error(_) => "Error".to_string(),
    }
  }
  fn to_symbol(&self) -> String {
    match self.kind {
      Token_Type::None => "None".to_string(),
      Token_Type::Function => "function".to_string(),
      Token_Type::And => "and".to_string(),
      Token_Type::Or => "or".to_string(),
      Token_Type::Xor => "xor".to_string(),
      Token_Type::Not => "not".to_string(),
      Token_Type::Do => "do".to_string(),
      Token_Type::End => "end".to_string(),
      Token_Type::Return => "return".to_string(),
      Token_Type::Break => "break".to_string(),
      Token_Type::Goto => "goto".to_string(),
      Token_Type::If => "if".to_string(),
      Token_Type::Then => "then".to_string(),
      Token_Type::Else => "else".to_string(),
      Token_Type::Else_If => "elseif".to_string(),
      Token_Type::For => "for".to_string(),
      Token_Type::While => "while".to_string(),
      Token_Type::Repeat => "repeat".to_string(),
      Token_Type::Until => "until".to_string(),
      Token_Type::Nil => "nil".to_string(),
      Token_Type::False => "false".to_string(),
      Token_Type::True => "true".to_string(),
      Token_Type::Method_Call => ":".to_string(),
      Token_Type::Label_Marker => "::".to_string(),
      Token_Type::Member => ".".to_string(),
      Token_Type::Concat => "..".to_string(),
      Token_Type::Ellipsis => "...".to_string(),
      Token_Type::Equal => "==".to_string(),
      Token_Type::Greater => ">".to_string(),
      Token_Type::Greater_Equal => ">=".to_string(),
      Token_Type::Lesser => "<".to_string(),
      Token_Type::Lesser_Equal => "<=".to_string(),
      Token_Type::Not_Equal => "~=".to_string(),
      Token_Type::Assign => "=".to_string(),
      Token_Type::Add => "+".to_string(),
      Token_Type::Subtract => "-".to_string(),
      Token_Type::Divide => "/".to_string(),
      Token_Type::Integer_Divide => "//".to_string(),
      Token_Type::Multiply => "*".to_string(),
      Token_Type::Exponent => "^".to_string(),
      Token_Type::Modulus => "%".to_string(),
      Token_Type::Length => "#".to_string(),
      Token_Type::Shift_Left => "<<".to_string(),
      Token_Type::Shift_Right => ">>".to_string(),
      Token_Type::Bitwise_And => "&".to_string(),
      Token_Type::Bitwise_Or => "|".to_string(),
      Token_Type::Bitwise_Not_Or_Xor => "~".to_string(),
      Token_Type::Int(ref aval) => format!("Int : '{}'", aval),
      Token_Type::Float(ref aval) => format!("Float : '{}'", aval),
      Token_Type::Str(ref aval, ref bval) => format!("Str : '{}' '{}'", aval, bval),
      Token_Type::Long_Str(ref aval, ref bval) => format!("Long_Str : '{}' '{}'", aval, bval),
      Token_Type::Local => "local".to_string(),
      Token_Type::Name(ref aval) => format!("Name : '{}'", aval),
      Token_Type::Element_Separator => ",".to_string(),
      Token_Type::Table_Open => "{".to_string(),
      Token_Type::Table_Close => "}".to_string(),
      Token_Type::Index_Open => "[".to_string(),
      Token_Type::Index_Close => "]".to_string(),
      Token_Type::Paren_Open => "(".to_string(),
      Token_Type::Paren_Close => ")".to_string(),
      Token_Type::Comment(ref aval) => format!("-- : '{}'", aval),
      Token_Type::Long_Comment(ref aval, ref bval) => format!("--[[]] : '{}' '{}'", aval, bval),
      Token_Type::Space(ref aval) => format!("space : '{}'", aval),
      Token_Type::New_Line(_) => format!("newline"),
      Token_Type::Error(_) => "error".to_string(),
    }
  }
}

struct Lex_State<'a> {
  input: &'a str,
  strider: std::str::CharIndices<'a>,
  curr: std::option::Option<(usize, char)>,
  line_id: usize,
  char_id: usize,
  line_start_char_id: usize,
}

fn is_new_line(thing: char) -> bool {
  thing == '\n' || thing == '\r'
}
fn is_name_start_char(thing: char) -> bool {
  thing == '_' || (thing.is_ascii() && thing.is_alphabetic())
}
fn is_name_mid_char(thing: char) -> bool {
  is_name_start_char(thing) || (thing.is_ascii() && thing.is_numeric())
}
fn hex_to_u(thing: char) -> Result<u32, &'static str> {
  match thing {
    '0' => Ok(0),
    '1' => Ok(1),
    '2' => Ok(2),
    '3' => Ok(3),
    '4' => Ok(4),
    '5' => Ok(5),
    '6' => Ok(6),
    '7' => Ok(7),
    '8' => Ok(8),
    '9' => Ok(9),
    'a' | 'A' => Ok(10),
    'b' | 'B' => Ok(11),
    'c' | 'C' => Ok(12),
    'd' | 'D' => Ok(13),
    'e' | 'E' => Ok(14),
    'f' | 'F' => Ok(15),
    _ => Err("invalid_hex_digit")
  }
}
fn add_slice(ret_str: &mut String, input: &str, slice_start: usize, slice_end: usize) {
  ret_str.push_str(&input[slice_start..slice_end]);
}

impl<'a> Lex_State<'a> {
  fn next(&mut self) {
    self.curr = self.strider.next();
  }
  fn next_space(&mut self) {
    while self.curr != None && !self.curr.unwrap().1.is_whitespace() {
      self.next();
    }
  }
  fn inc_line_number(&mut self) {
    // These count as one line each:
    // \n, \r, \r\n, \n\r
    // These count as two lines each:
    // \n\n, \r\r
    let old = self.curr.unwrap().1;
    self.next();
    if self.curr == None {
      self.line_id = self.line_id + 1;
    } else {
      let thing = self.curr.unwrap().1;
      if is_new_line(thing) {
        if thing == old {
          self.line_id = self.line_id + 1;
          self.line_start_char_id = self.curr.unwrap().0;
        } else {
          self.next();
        }
      } else {
        self.line_id = self.line_id + 1;
        self.line_start_char_id = self.curr.unwrap().0;
      }
    }
  }
  fn count_long_string_sep(&mut self) -> i32 {
    let mut sep_size = 0;
    while self.curr != None {
      match self.curr.unwrap().1 {
        '=' => {
          sep_size = sep_size + 1;
          self.next();
        },
        _ => break,
      }
    }
    sep_size
  }
  fn read_long_string(&mut self) -> Result<(String, i32), &'static str> {
    // Current position is the char after the first '[' of "[[" or "[=[".
    // Count '=' until the second [ is found.
    // Error on anything else.
    let sep_size = self.count_long_string_sep();
    if self.curr == None {
      return Err("long_string_malformed_beginning")
    }
    if self.curr.unwrap().1 != '[' {
      return Err("long_string_malformed_beginning")
    }
    self.next();
    if self.curr == None {
      return Err("long_string_malformed_beginning")
    }
    // Skip a leading newline.
    if is_new_line(self.curr.unwrap().1) {
      self.next();
      if self.curr == None {
        return Err("long_string_malformed_beginning")
      }
    }
    // Lua's read_long_string converts any newline in a long string to \n
    // disregarding type.  So slicing the input string from the start of
    // the long string to the end would cause this lexer to produce
    // different results for long strings that contain newlines.
    // Instead, each line will be copied as a slice.
    let mut ret_str = String::new();
    let mut slice_start = self.curr.unwrap().0;
    while self.curr != None {
      match self.curr.unwrap().1 {
        ']' => {
          let end_sep_start = self.curr.unwrap().0;
          self.next();
          if self.curr == None {
            return Err("long_string_malformed_ending")
          }
          let end_size = self.count_long_string_sep();
          if self.curr == None {
            return Err("long_string_malformed_ending")
          }
          if end_size == sep_size && self.curr.unwrap().1 == ']' {
            ret_str.push_str(&self.input[slice_start..end_sep_start]);
            self.next();
            return Ok((ret_str, sep_size))
          } else {
            self.next();
          }
        },
        '\n' | '\r' => {
          let slice_end = self.curr.unwrap().0;
          ret_str.push_str(&self.input[slice_start..slice_end]);
          ret_str.push('\n');
          self.inc_line_number();
          if self.curr == None {
            return Err("long_string_malformed_ending")
          }
          slice_start = self.curr.unwrap().0;
        },
        _ => {
          self.next();
        }
      }
    }
    return Err("long_string_malformed_ending")
  }
  fn read_hex_esc(&mut self) -> Result<char, &'static str> {
    self.next();
    // Must be exactly two hex digits in the escape sequence.
    if self.curr == None {
      return Err("unfinished_hex_escape")
    }
    let escape_start = self.curr.unwrap().0;
    self.next();
    if self.curr == None {
      return Err("unfinished_hex_escape")
    }
    self.next();
    if self.curr == None {
      return Err("unfinished_hex_escape")
    }
    let escape_end = self.curr.unwrap().0;
    let result = u8::from_str_radix(&self.input[escape_start..escape_end], 16);
    match result {
      Ok(res) => return Ok(char::from(res)),
      // _why is unused because I decided to return a static lifetime
      // string literal, so it can't be formatted into the error.  Maybe
      // that was a bad choice.
      Err(_why) => return Err("hex_escape_error")
    }
  }
  fn read_utf_esc(&mut self) -> Result<String, &'static str> {
    // current is on the u in "\u{XXX}".  The brackets are required and XXX
    // is one or more hexadecimal digits, up to 8 (32 bits).
    self.next();
    if self.curr == None || self.curr.unwrap().1 != '{' {
      return Err("utf8_escape_missing_bracket")
    }
    self.next();
    if self.curr == None {
      return Err("utf8_escape_missing_digits")
    }
    let mut result: u32 = 0;
    while self.curr != None && self.curr.unwrap().1 != '}' {
      match hex_to_u(self.curr.unwrap().1) {
        Ok(res) => {
          result = (result << 4) | res;
          if result > 0x10ffff {
            return Err("utf8_value_too_large")
          }
        },
        Err(why) => return Err(why)
      }
      self.next();
    }
    self.next();
    let mut escbytes: Vec<u8> = Vec::new();
    if result < 0x80 { // ascii
      escbytes.push(result as u8);
    } else {
      let mut first_byte_bits = 0x3f;
      while result > first_byte_bits {
        let byte = (0x80 | (result & 0x3f)) as u8;
        // Insert at the front because they're created in reverse order.
        escbytes.insert(0, byte);
        result >>= 6; // remove added bits
        first_byte_bits >>= 1; // one less bit available in first byte
      }
      let first_byte = ((!first_byte_bits << 1) | result) as u8;
      escbytes.insert(0, first_byte);
    }
    match String::from_utf8(escbytes) {
      Ok(res) => return Ok(res),
      Err(_why) => return Err("invalid_utf8")
    }
  }
  fn read_dec_esc(&mut self) -> Result<char, &'static str> {
    self.next();
    let mut result: u32 = 0;
    let mut read = 0;
    while self.curr != None {
      match self.curr.unwrap().1.to_digit(10) {
        Some(digit) => {
          result = (result * 10) + digit;
          read = read + 1;
          self.next();
          if read >= 3 { break; }
        },
        None => break
      }
    }
    if result > u8::max_value() as u32 {
      Err("decimal_escape_too_large")
    } else {
      Ok(char::from(result as u8))
    }
  }
  fn read_string(&mut self) -> Result<(String, char), &'static str> {
    let delim = self.curr.unwrap().1;
    let mut ret_str = String::new();
    self.next();
    if self.curr == None {
      return Err("unfinished_string")
    }
    let mut slice_start = self.curr.unwrap().0;
    while self.curr != None {
      match self.curr.unwrap().1 {
        '\n' | '\r' => {
          return Err("unfinished_string")
        },
        '\\' => {
          add_slice(&mut ret_str, self.input, slice_start, self.curr.unwrap().0);
          self.next();
          if self.curr == None {
            return Err("unfinished_string")
          }
          let mut advance = true;
          // TODO: Handle a, b, f, v
          match self.curr.unwrap().1 {
            'n' => ret_str.push('\n'),
            'r' => ret_str.push('\r'),
            't' => ret_str.push('\t'),
            'x' => {
              match self.read_hex_esc() {
                Ok(res) => ret_str.push(res),
                Err(why) => {
                  return Err(why)
                }
              }
              advance= false;
            },
            'u' => {
              match self.read_utf_esc() {
                Ok(res) => ret_str.push_str(&res[..]),
                Err(why) => {
                  return Err(why)
                }
              }
              advance= false;
            },
            '\n' | '\r' => {
              self.inc_line_number();
              advance = false;
            },
            '\\' | '\"' | '\'' => ret_str.push(self.curr.unwrap().1),
            'z' => {
              self.next();
              while self.curr != None && self.curr.unwrap().1.is_whitespace() {
                if is_new_line(self.curr.unwrap().1) {
                  self.inc_line_number();
                } else {
                  self.next();
                }
              }
              advance= false;
            },
            _ => {
              if self.curr.unwrap().1.is_digit(10) {
                match self.read_dec_esc() {
                  Ok(res) => ret_str.push(res),
                  Err(why) => {
                    return Err(why)
                  }
                }
                advance= false;
              } else {
                return Err("invalid_escape_sequence")
              }
            },
          }
          if advance {
            self.next();
          }
          match self.curr {
            None => return Err("unfinished_string"),
            Some((pos, _)) => slice_start = pos,
          }
        },
        _ => {
          if self.curr.unwrap().1 == delim {
            add_slice(&mut ret_str, self.input, slice_start, self.curr.unwrap().0);
            self.next();
            return Ok((ret_str, delim))
          } else {
            self.next()
          }
        },
      }
    }
    return Err("unfinished_string")
  }
  fn read_numeral(&mut self, leading_decimal: bool, decimal_pos : usize) -> Token_Type {
    let mut radix = 10;
    let mut is_float;
    let mut numeral_start;
    if leading_decimal {
      is_float = true;
      numeral_start = decimal_pos;
    } else {
      is_float = false;
      numeral_start = self.curr.unwrap().0;
    }
    if self.curr.unwrap().1 == '0' {
      self.next();
      match self.curr {
        Some((_, 'x')) | Some((_, 'X')) => {
          radix = 16;
          self.next();
          if self.curr == None {
            return Token_Type::Error("invalid_numeral_prefix".to_string())
          } else {
            numeral_start = self.curr.unwrap().0;
          }
        },
        Some((_, _)) => {},
        None => return Token_Type::Int(0)
      }
    }
    while self.curr != None {
      match self.curr.unwrap().1 {
        '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'e' | 'E' => {
          self.next();
        },
        '.' => {
          if is_float {
            break;
          } else {
            is_float = true;
            self.next();
          }
        },
        'a' | 'A' | 'b' | 'B' | 'c' | 'C' | 'd' | 'D' | 'f' | 'F' => {
          if radix == 16 {
            self.next();
          } else {
            break;
          }
        },
        _ => {
          break;
        }
      }
    }
    let numeral_end = if self.curr == None {
      self.input.len()
    } else {
      self.curr.unwrap().0
    };
    let numeral = self.input[numeral_start..numeral_end].to_string();
    if is_float {
      match f64::from_str(&numeral) {
        Ok(res) => return Token_Type::Float(res),
        Err(_why) => return Token_Type::Error(format!("invalid_float_numeral {}", numeral))
      }
    } else {
      match i64::from_str_radix(&numeral, radix) {
        Ok(res) => return Token_Type::Int(res),
        Err(_why) => return Token_Type::Error(format!("invalid_int_numeral {}", numeral))
      }
    }
  }
}

pub fn lex_str(input: &str) -> Vec<Token> {
  let mut state = Lex_State{
    input,
    strider: input.char_indices(), curr: None,
    line_id: 1, char_id: 0,
    line_start_char_id: 0,
  };
  let mut name_tokens = HashMap::new();
  name_tokens.insert("function".to_string(), Token_Type::Function);
  name_tokens.insert("and".to_string(), Token_Type::And);
  name_tokens.insert("or".to_string(), Token_Type::Or);
  name_tokens.insert("not".to_string(), Token_Type::Not);
  name_tokens.insert("do".to_string(), Token_Type::Do);
  name_tokens.insert("end".to_string(), Token_Type::End);
  name_tokens.insert("return".to_string(), Token_Type::Return);
  name_tokens.insert("break".to_string(), Token_Type::Break);
  name_tokens.insert("goto".to_string(), Token_Type::Goto);
  name_tokens.insert("if".to_string(), Token_Type::If);
  name_tokens.insert("then".to_string(), Token_Type::Then);
  name_tokens.insert("else".to_string(), Token_Type::Else);
  name_tokens.insert("elseif".to_string(), Token_Type::Else_If);
  name_tokens.insert("for".to_string(), Token_Type::For);
  name_tokens.insert("while".to_string(), Token_Type::While);
  name_tokens.insert("repeat".to_string(), Token_Type::Repeat);
  name_tokens.insert("until".to_string(), Token_Type::Until);
  name_tokens.insert("nil".to_string(), Token_Type::Nil);
  name_tokens.insert("false".to_string(), Token_Type::False);
  name_tokens.insert("true".to_string(), Token_Type::True);
  name_tokens.insert("local".to_string(), Token_Type::Local);
  state.next();
  let mut result: Vec<Token> = Vec::new();
  while state.curr != None {
    let pos_char = state.curr.unwrap();
    state.char_id = pos_char.0;
    let mut in_progress = Token{
      kind: Token_Type::None,
      line_id: state.line_id,
      char_id: state.char_id - state.line_start_char_id,
    };
    match pos_char.1 {
      ' ' | '\t' => {
        in_progress.kind = Token_Type::Space(pos_char.1.to_string());
        // TODO: C lua handled \f and \v, find the equivalent in rust and handle them.
        state.next();
      },
      '\n' | '\r' => {
        in_progress.kind = Token_Type::New_Line(pos_char.1.to_string());
        state.inc_line_number();
      },
      '-' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Subtract;
        } else {
          if state.curr.unwrap().1 == '-' {
            state.next();
            if state.curr == None {
              in_progress.kind = Token_Type::Comment("--".to_string());
            } else {
              if state.curr.unwrap().1 == '[' {
                state.next();
                match state.read_long_string() {
                  Ok(str_info) => {
                    in_progress.kind = Token_Type::Long_Comment(str_info.0, str_info.1);
                  },
                  Err(why) => {
                    in_progress.kind = Token_Type::Error(why.to_string());
                    state.next_space();
                  }
                }
              } else {
                while state.curr != None && !is_new_line(state.curr.unwrap().1) {
                  state.next();
                }
                let token_end = {
                  if state.curr == None {
                    input.len()
                  } else {
                    state.curr.unwrap().0
                  }
                };
                in_progress.kind = Token_Type::Comment(input[state.char_id..token_end].to_string());
              }
            }
          } else {
            in_progress.kind = Token_Type::Subtract;
          }
        }
      },
      '[' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Index_Open;
        } else {
          if state.curr.unwrap().1 == '[' || state.curr.unwrap().1 == '=' {
            match state.read_long_string() {
              Ok(str_info) => {
                in_progress.kind = Token_Type::Long_Str(str_info.0, str_info.1);
              },
              Err(why) => {
                in_progress.kind = Token_Type::Error(why.to_string());
                state.next_space();
              }
            }
          } else {
            in_progress.kind = Token_Type::Index_Open;
          }
        }
      },
      ']' => {
        in_progress.kind = Token_Type::Index_Close;
        state.next();
      },
      '=' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Assign;
        } else {
          if state.curr.unwrap().1 == '=' {
            in_progress.kind = Token_Type::Equal;
            state.next();
          } else {
            in_progress.kind = Token_Type::Assign;
          }
        }
      },
      '<' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Lesser;
        } else {
          if state.curr.unwrap().1 == '=' {
            in_progress.kind = Token_Type::Lesser_Equal;
            state.next();
          } else {
            in_progress.kind = Token_Type::Lesser;
          }
        }
      },
      '>' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Greater;
        } else {
          if state.curr.unwrap().1 == '=' {
            in_progress.kind = Token_Type::Greater_Equal;
            state.next();
          } else {
            in_progress.kind = Token_Type::Greater;
          }
        }
      },
      '/' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Divide;
        } else {
          if state.curr.unwrap().1 == '/' {
            in_progress.kind = Token_Type::Integer_Divide;
            state.next();
          } else {
            in_progress.kind = Token_Type::Divide;
          }
        }
      },
      '~' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Bitwise_Not_Or_Xor;
        } else {
          if state.curr.unwrap().1 == '=' {
            in_progress.kind = Token_Type::Not_Equal;
            state.next();
          } else {
            in_progress.kind = Token_Type::Bitwise_Not_Or_Xor;
          }
        }
      },
      ':' => {
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Method_Call;
        } else {
          if state.curr.unwrap().1 == ':' {
            in_progress.kind = Token_Type::Label_Marker;
            state.next();
          } else {
            in_progress.kind = Token_Type::Method_Call;
          }
        }
      },
      '"' | '\'' => {
        match state.read_string() {
          Ok(str_info) => in_progress.kind = Token_Type::Str(str_info.0, str_info.1),
          Err(why) => {
            in_progress.kind = Token_Type::Error(why.to_string());
            state.next_space();
          }
        }
      },
      '.' => {
        let decimal_pos = state.curr.unwrap().0;
        state.next();
        if state.curr == None {
          in_progress.kind = Token_Type::Member;
        } else {
          match state.curr.unwrap().1 {
            '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => {
              in_progress.kind = state.read_numeral(true, decimal_pos);
            },
            '.' => {
              state.next();
              if state.curr == None || state.curr.unwrap().1 != '.' {
                in_progress.kind = Token_Type::Concat;
              } else {
                in_progress.kind = Token_Type::Ellipsis;
                state.next();
              }
            },
            _ => {
              in_progress.kind = Token_Type::Member;
            }
          }
        }
      },
      '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => {
        in_progress.kind = state.read_numeral(false, 0);
      },
      '+' => {in_progress.kind = Token_Type::Add; state.next();},
      '*' => {in_progress.kind = Token_Type::Multiply; state.next();},
      '%' => {in_progress.kind = Token_Type::Modulus; state.next();},
      '&' => {in_progress.kind = Token_Type::Bitwise_And; state.next();},
      '^' => {in_progress.kind = Token_Type::Exponent; state.next();},
      '|' => {in_progress.kind = Token_Type::Bitwise_Or; state.next();},
      '#' => {in_progress.kind = Token_Type::Length; state.next();},
      ',' | ';' => {in_progress.kind = Token_Type::Element_Separator; state.next();},
      '(' => {in_progress.kind = Token_Type::Paren_Open; state.next();},
      ')' => {in_progress.kind = Token_Type::Paren_Close; state.next();},
      '{' => {in_progress.kind = Token_Type::Table_Open; state.next();},
      '}' => {in_progress.kind = Token_Type::Table_Close; state.next();},
      _ => {
        if is_name_start_char(state.curr.unwrap().1) {
          let name_start = state.curr.unwrap().0;
          state.next();
          while state.curr != None {
            if is_name_mid_char(state.curr.unwrap().1) {
              state.next();
            } else {
              break;
            }
          }
          let name_end = if state.curr == None {
            state.input.len()
          } else {
            state.curr.unwrap().0
          };
          let name = state.input[name_start..name_end].to_string();
          match name_tokens.get(&name) {
            Some(tokty) => in_progress.kind = tokty.clone(),
            None => in_progress.kind = Token_Type::Name(name),
          }
        } else {
          state.next();
        }
      }
    }
    result.push(in_progress);
  }
  result
}
