Skeleton scanner impl

forge-script-lang/src/lib.rs

 mod error;
 mod lexer;
 mod parser;
+mod pratt;
 pub mod runtime;
 mod utilities;
 

forge-script-lang/src/pratt/mod.rs

+mod parser;
+
+#[cfg(test)]
+mod test_cases;

forge-script-lang/src/pratt/parser.rs

+use crate::lexer::ScriptTokenType;
+
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct PositionState {
+	offset: usize,
+	line: usize,
+	column: usize,
+}
+
+impl Default for PositionState {
+	fn default() -> Self {
+		Self {
+			offset: 0,
+			column: 0,
+			line: 1,
+		}
+	}
+}
+
+impl PositionState {
+	pub fn advance(&mut self, amount: usize) {
+		self.offset += amount;
+		self.column += amount;
+	}
+
+	pub fn new_line(&mut self) {
+		self.offset += 1;
+		self.line += 1;
+		self.column = 0;
+	}
+
+	pub fn offset(&self) -> usize {
+		self.offset
+	}
+	pub fn line(&self) -> usize {
+		self.line
+	}
+	pub fn column(&self) -> usize {
+		self.column
+	}
+}
+
+#[derive(Clone, Copy, PartialEq, Debug, Default)]
+pub struct ScanningState {
+	lexeme_start: usize,
+	lexeme_current: usize,
+}
+
+impl ScanningState {
+	pub fn new(idx: usize) -> Self {
+		ScanningState {
+			lexeme_start: idx,
+			lexeme_current: idx,
+		}
+	}
+
+	pub fn with_width(idx: usize, width: usize) -> Self {
+		ScanningState {
+			lexeme_start: idx,
+			lexeme_current: idx + width,
+		}
+	}
+
+	pub fn advance(&mut self, amount: usize) {
+		self.lexeme_current += amount;
+	}
+
+	pub fn start(&self) -> usize {
+		self.lexeme_start
+	}
+	pub fn current(&self) -> usize {
+		self.lexeme_current
+	}
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct Scanner<'a> {
+	source: &'a str,
+	position: PositionState,
+	scan_state: ScanningState,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum ScannerErrorKind<'a> {
+	UnexpectedEof,
+	UnexpectedCharacter { span: TokenSpan<'a> },
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ScannerError<'a> {
+	pub kind: ScannerErrorKind<'a>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub struct TokenSpan<'a> {
+	pub position: PositionState,
+	pub length: usize,
+	pub source: &'a str,
+}
+
+impl<'a> From<Scanner<'a>> for TokenSpan<'a> {
+	fn from(value: Scanner<'a>) -> TokenSpan<'a> {
+		TokenSpan {
+			source: value.source,
+			position: value.position,
+			length: value
+				.scan_state
+				.lexeme_current
+				.saturating_sub(value.scan_state.lexeme_start),
+		}
+	}
+}
+
+#[derive(Clone, Debug)]
+pub struct ScannerToken<'a> {
+	pub location: TokenSpan<'a>,
+	pub token: ScriptTokenType<'a>,
+}
+
+pub type ScannerResult<'a> = Result<ScannerToken<'a>, ScannerError<'a>>;
+
+impl<'a> Scanner<'a> {
+	pub fn new(source: &str) -> Scanner {
+		Scanner {
+			source,
+			position: PositionState::default(),
+			scan_state: ScanningState::default(),
+		}
+	}
+
+	pub fn is_finished(&self) -> bool {
+		self.position.offset == self.source.len()
+	}
+
+	pub fn scan_token(&mut self) -> Result<ScannerToken<'a>, ScannerError> {
+		if self.is_finished() {
+			Ok(ScannerToken {
+				token: ScriptTokenType::Eof,
+				location: TokenSpan::from(*self),
+			})
+		} else {
+			Err(ScannerError {
+				kind: ScannerErrorKind::UnexpectedEof,
+			})
+		}
+	}
+}

forge-script-lang/src/pratt/test_cases.rs

+use crate::lexer::ScriptTokenType;
+use crate::pratt::parser::{Scanner, ScannerError, ScannerResult};
+use test_case::test_case;
+
+#[test_case("1 + 2" => Ok(ScriptTokenType::Integer(1)) ; "expects integer")]
+#[test_case("print 1 + 2" => Ok(ScriptTokenType::Print) ; "expects print")]
+#[test_case("\"Foo\"" => matches Ok(ScriptTokenType::OwnedString(_)) ; "expects string")]
+#[test_case("" => Ok(ScriptTokenType::Eof) ; "expects eof")]
+fn next_token(source: &'static str) -> Result<ScriptTokenType<'static>, ScannerError> {
+	let mut scanner = Scanner::new(source);
+	scanner.scan_token().map(|t| t.token)
+}