/**
 * Architect language lexer.
 * Converts a raw document text string to a flat Token[] in document order.
 *
 * Design notes:
 * - Stateful: tracks hash mode line by line (HASH-ON is the default).
 * - In HASH-ON mode:  lines starting with '#' are code; all other lines are output.
 * - In HASH-OFF mode: all lines are code; output lines are prefixed with '>>' (OICC).
 * - The '#' prefix on code lines in HASH-ON mode is consumed as HASH_PREFIX and is
 *   NOT passed to the parser — the parser never sees a raw '#' at position 0.
 * - Output lines are emitted as a single OUTPUT_TEXT token (not tokenised further).
 * - Every character in the source is accounted for in the token stream.
 * - Case is preserved in token.value; keyword recognition uses uppercase comparison.
 */

import { TokenType } from "./tokenTypes.js";

// Keywords recognised by the language (from constants.js, kept local to avoid a dependency).
// Includes multi-word tokens treated as single keywords (HASH-ON, HASH-OFF, END-OF-CODE).
const KEYWORDS = new Set([
    "AND", "OR", "NOT", "TRUE", "FALSE",
    "IF", "ENDIF", "ELSE",
    "DO", "ENDDO", "WHILE", "FOREVER", "UNTIL", "BREAK", "CONTINUE", "TO", "BY", "MOD",
    "CASE", "ENDCASE", "WHEN", "OTHERWISE", "VALUE", "OF",
    "FUNCTION", "ENDFUNCTION", "RETURN",
    "MACRO",
    "VAR", "SET", "OPTION", "HASH-ON", "HASH-OFF", "NEWINTERPRETER",
    "INCLUDE", "INCLUDEONCE", "INCLUDETEST", "REQUIRE",
    "EXIT", "END-OF-CODE"
]);

// Type keywords — recognised as TYPE_KEYWORD rather than KEYWORD.
const TYPE_KEYWORDS = new Set([
    "STRING", "SHORT", "LONG", "DOUBLE", "BOOLEAN", "DATE", "TIME", "INTEGER", "BIGINT"
]);

// Default output indicator (OICC) used in HASH-OFF mode.
// The language allows this to be overridden via OPTION OICC = "..."; for now we treat
// ">>" as the canonical default. Full OICC override support is a Phase 3 concern.
const DEFAULT_OICC = ">>";

/**
 * Tokenise a full document text string.
 *
 * @param {string} text - Raw document content (may use \n or \r\n line endings).
 * @returns {Token[]} - Flat array of tokens in document order, ending with a single EOF.
 */
function tokenize(text) {
    const tokens = [];
    const lines = splitLines(text);

    // Lexer state
    let hashMode = true; // true = HASH-ON (default), false = HASH-OFF

    for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
        const { content, newline } = lines[lineIndex];
        tokenizeLine(content, newline, lineIndex, hashMode, tokens);

        // After tokenising, update hash mode if this line contained a directive.
        // We check the last non-NEWLINE, non-WHITESPACE tokens for the directive keyword.
        hashMode = updatedHashMode(hashMode, tokens);
    }

    tokens.push(makeToken(TokenType.EOF, "", lines.length, 0));
    return tokens;
}

// ---------------------------------------------------------------------------
// Line splitting
// ---------------------------------------------------------------------------

/**
 * Split text into lines, preserving the newline character(s) separately.
 * @param {string} text
 * @returns {{ content: string, newline: string }[]}
 */
function splitLines(text) {
    const result = [];
    let start = 0;

    for (let i = 0; i < text.length; i++) {
        if (text[i] === "\r" && text[i + 1] === "\n") {
            result.push({ content: text.slice(start, i), newline: "\r\n" });
            start = i + 2;
            i += 1; // skip the \n
        } else if (text[i] === "\n") {
            result.push({ content: text.slice(start, i), newline: "\n" });
            start = i + 1;
        }
    }

    // Last line (may have no trailing newline)
    result.push({ content: text.slice(start), newline: "" });
    return result;
}

// ---------------------------------------------------------------------------
// Line-level classification and tokenisation
// ---------------------------------------------------------------------------

/**
 * Tokenise a single line and push results into the tokens array.
 */
function tokenizeLine(content, newline, lineIndex, hashMode, tokens) {
    let pos = 0; // current character position within `content`

    if (hashMode) {
        // HASH-ON: a line starting with '#' (optionally preceded by whitespace) is code.
        // All other lines are output text.
        const firstNonWs = content.search(/\S/);
        const isCodeLine = firstNonWs !== -1 && content[firstNonWs] === "#";

        if (!isCodeLine) {
            // Entire line is output text (even if empty)
            tokens.push(makeToken(TokenType.OUTPUT_TEXT, content, lineIndex, 0));
            if (newline) {
                tokens.push(makeToken(TokenType.NEWLINE, newline, lineIndex, content.length));
            }
            return;
        }

        // Leading whitespace before the '#'
        if (firstNonWs > 0) {
            tokens.push(makeToken(TokenType.WHITESPACE, content.slice(0, firstNonWs), lineIndex, 0));
            pos = firstNonWs;
        }

        // Consume the '#' as HASH_PREFIX
        tokens.push(makeToken(TokenType.HASH_PREFIX, "#", lineIndex, pos));
        pos += 1;

        // Now tokenise the rest of the line as code
        tokenizeCodeContent(content, lineIndex, pos, tokens);
    } else {
        // HASH-OFF: all lines are code unless they start with the OICC prefix (default ">>").
        const trimmed = content.trimStart();
        if (trimmed.startsWith(DEFAULT_OICC)) {
            // Output line
            tokens.push(makeToken(TokenType.OUTPUT_TEXT, content, lineIndex, 0));
            if (newline) {
                tokens.push(makeToken(TokenType.NEWLINE, newline, lineIndex, content.length));
            }
            return;
        }

        // Code line — no HASH_PREFIX emitted in HASH-OFF mode
        tokenizeCodeContent(content, lineIndex, 0, tokens);
    }

    if (newline) {
        tokens.push(makeToken(TokenType.NEWLINE, newline, lineIndex, content.length));
    }
}

// ---------------------------------------------------------------------------
// Code content tokeniser (operates on a single line after hash classification)
// ---------------------------------------------------------------------------

/**
 * Tokenise the code portion of a line, starting at `startPos`.
 * Pushes tokens into the array. Does NOT emit a NEWLINE (caller handles that).
 */
function tokenizeCodeContent(line, lineIndex, startPos, tokens) {
    let pos = startPos;
    const len = line.length;

    while (pos < len) {
        const ch = line[pos];

        // --- Whitespace ---
        if (ch === " " || ch === "\t") {
            let end = pos + 1;
            while (end < len && (line[end] === " " || line[end] === "\t")) {
                end++;
            }
            tokens.push(makeToken(TokenType.WHITESPACE, line.slice(pos, end), lineIndex, pos));
            pos = end;
            continue;
        }

        // --- Comment: // ---
        if (ch === "/" && line[pos + 1] === "/") {
            // Rest of line is a comment
            tokens.push(makeToken(TokenType.COMMENT, line.slice(pos), lineIndex, pos));
            return; // nothing after a comment on the same line
        }

        // --- String literals: "..." or '...' ---
        if (ch === "\"" || ch === "'") {
            const quote = ch;
            let end = pos + 1;
            while (end < len) {
                if (line[end] === quote) {
                    end++;
                    break;
                }
                // No escape sequences in Architect — a closing quote terminates the literal.
                end++;
            }
            tokens.push(makeToken(TokenType.STRING_LITERAL, line.slice(pos, end), lineIndex, pos));
            pos = end;
            continue;
        }

        // --- Date literals, time literals, and number literals (all start with a digit) ---
        if (ch >= "0" && ch <= "9") {
            // Check for date literal: D{1,2}/D{1,2}/D{4} (no surrounding whitespace)
            const dateEnd = tryReadDateLiteral(line, pos);
            if (dateEnd !== -1) {
                tokens.push(makeToken(TokenType.DATE_LITERAL, line.slice(pos, dateEnd), lineIndex, pos));
                pos = dateEnd;
                continue;
            }
            // Check for time literal: D{1,2}:D{1,2}:D{1,2}:D{1,2} (no spaces, 1-2 digits each)
            const timeEnd = tryReadTimeLiteral(line, pos);
            if (timeEnd !== -1) {
                tokens.push(makeToken(TokenType.TIME_LITERAL, line.slice(pos, timeEnd), lineIndex, pos));
                pos = timeEnd;
                continue;
            }
            // Number literal (integer or decimal)
            let end = pos + 1;
            while (end < len && (line[end] >= "0" && line[end] <= "9")) {
                end++;
            }
            // Allow one decimal point
            if (end < len && line[end] === "." && end + 1 < len && line[end + 1] >= "0" && line[end + 1] <= "9") {
                end++;
                while (end < len && line[end] >= "0" && line[end] <= "9") {
                    end++;
                }
            }
            tokens.push(makeToken(TokenType.NUMBER_LITERAL, line.slice(pos, end), lineIndex, pos));
            pos = end;
            continue;
        }

        // --- Word tokens: keywords, type keywords, identifiers ---
        // Architect identifiers: [A-Za-z_][A-Za-z0-9_]*
        // Also handle hyphenated keywords like HASH-ON, HASH-OFF, END-OF-CODE.
        if (isWordStart(ch)) {
            let end = pos + 1;
            while (end < len && isWordContinue(line[end])) {
                end++;
            }

            // Check for hyphenated keyword continuation (e.g. HASH-ON, HASH-OFF, END-OF-CODE).
            // Greedily extend across hyphens until we match a known keyword.
            if (end < len && line[end] === "-") {
                let extEnd = end;
                while (extEnd < len && line[extEnd] === "-") {
                    const segment = readWordFrom(line, extEnd + 1);
                    if (!segment) break;
                    extEnd = extEnd + 1 + segment.length;
                    const candidate = line.slice(pos, extEnd);
                    if (KEYWORDS.has(candidate.toUpperCase())) {
                        end = extEnd;
                        break;
                    }
                }
            }

            const word = line.slice(pos, end);
            const upper = word.toUpperCase();

            let type;
            if (KEYWORDS.has(upper)) {
                type = TokenType.KEYWORD;
            } else if (TYPE_KEYWORDS.has(upper)) {
                type = TokenType.TYPE_KEYWORD;
            } else {
                type = TokenType.IDENTIFIER;
            }

            tokens.push(makeToken(type, word, lineIndex, pos));
            pos = end;
            continue;
        }

        // --- Double-character operators that must be checked before single-char ---
        if (ch === "*" && line[pos + 1] === "*") {
            tokens.push(makeToken(TokenType.DOUBLE_STAR, "**", lineIndex, pos));
            pos += 2;
            continue;
        }

        if (ch === "?" && line[pos + 1] === "?") {
            tokens.push(makeToken(TokenType.DOUBLE_QUESTION, "??", lineIndex, pos));
            pos += 2;
            continue;
        }

        if (ch === "<" && line[pos + 1] === ">") {
            tokens.push(makeToken(TokenType.NOT_EQ, "<>", lineIndex, pos));
            pos += 2;
            continue;
        }

        if (ch === "<" && line[pos + 1] === "=") {
            tokens.push(makeToken(TokenType.LT_EQ, "<=", lineIndex, pos));
            pos += 2;
            continue;
        }

        if (ch === ">" && line[pos + 1] === "=") {
            tokens.push(makeToken(TokenType.GT_EQ, ">=", lineIndex, pos));
            pos += 2;
            continue;
        }

        // --- Continuation operator: ... ---
        if (ch === "." && pos + 2 < len && line[pos + 1] === "." && line[pos + 2] === ".") {
            tokens.push(makeToken(TokenType.CONTINUATION, "...", lineIndex, pos));
            pos += 3;
            continue;
        }

        // --- Single-character tokens ---
        const single = singleCharToken(ch, lineIndex, pos);
        if (single) {
            tokens.push(single);
            pos += 1;
            continue;
        }

        // --- Unknown character ---
        tokens.push(makeToken(TokenType.UNKNOWN, ch, lineIndex, pos));
        pos += 1;
    }
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

function isWordStart(ch) {
    return (ch >= "A" && ch <= "Z") || (ch >= "a" && ch <= "z") || ch === "_";
}

function isWordContinue(ch) {
    return isWordStart(ch) || (ch >= "0" && ch <= "9");
}

/**
 * Read a word (identifier-like) from a given position in a line string.
 * Returns the word characters without consuming any state.
 */
function readWordFrom(line, start) {
    let end = start;
    while (end < line.length && isWordContinue(line[end])) {
        end++;
    }
    return line.slice(start, end);
}

function singleCharToken(ch, lineIndex, pos) {
    switch (ch) {
        case ":": return makeToken(TokenType.COLON,     ch, lineIndex, pos);
        case ",": return makeToken(TokenType.COMMA,     ch, lineIndex, pos);
        case "(": return makeToken(TokenType.LPAREN,    ch, lineIndex, pos);
        case ")": return makeToken(TokenType.RPAREN,    ch, lineIndex, pos);
        case "[": return makeToken(TokenType.LBRACKET,  ch, lineIndex, pos);
        case "]": return makeToken(TokenType.RBRACKET,  ch, lineIndex, pos);
        case "=": return makeToken(TokenType.EQUALS,    ch, lineIndex, pos);
        case "+": return makeToken(TokenType.PLUS,      ch, lineIndex, pos);
        case "-": return makeToken(TokenType.MINUS,     ch, lineIndex, pos);
        case "*": return makeToken(TokenType.STAR,      ch, lineIndex, pos);
        case "/": return makeToken(TokenType.SLASH,     ch, lineIndex, pos);
        case "?": return makeToken(TokenType.QUESTION,  ch, lineIndex, pos);
        case "#": return makeToken(TokenType.HASH,      ch, lineIndex, pos);
        case "&": return makeToken(TokenType.AMPERSAND, ch, lineIndex, pos);
        case "|": return makeToken(TokenType.PIPE,      ch, lineIndex, pos);
        case "^": return makeToken(TokenType.CARET,     ch, lineIndex, pos);
        case "~": return makeToken(TokenType.TILDE,     ch, lineIndex, pos);
        case "%": return makeToken(TokenType.PERCENT,   ch, lineIndex, pos);
        case "<": return makeToken(TokenType.LT,        ch, lineIndex, pos);
        case ">": return makeToken(TokenType.GT,        ch, lineIndex, pos);
        case ".": return makeToken(TokenType.DOT,       ch, lineIndex, pos);
        default:  return null;
    }
}

/**
 * Try to read a DATE_LITERAL starting at pos.
 * Pattern: 1-2 digits, '/', 1-2 digits, '/', exactly 4 digits (no whitespace).
 * Returns end position (exclusive) on match, -1 otherwise.
 */
function tryReadDateLiteral(line, pos) {
    const len = line.length;
    let i = pos;

    // Day: 1-2 digits
    if (i >= len || !(line[i] >= "0" && line[i] <= "9")) return -1;
    i++;
    if (i < len && line[i] >= "0" && line[i] <= "9") i++;

    if (i >= len || line[i] !== "/") return -1;
    i++;

    // Month: 1-2 digits
    if (i >= len || !(line[i] >= "0" && line[i] <= "9")) return -1;
    i++;
    if (i < len && line[i] >= "0" && line[i] <= "9") i++;

    if (i >= len || line[i] !== "/") return -1;
    i++;

    // Year: exactly 4 digits
    const yearStart = i;
    while (i < len && line[i] >= "0" && line[i] <= "9") i++;
    if (i - yearStart !== 4) return -1;

    // Must not be followed by more digits
    if (i < len && line[i] >= "0" && line[i] <= "9") return -1;

    return i;
}

/**
 * Try to read a TIME_LITERAL starting at pos.
 * Pattern: 4 colon-separated components, each 1-2 digits (no whitespace).
 * Returns end position (exclusive) on match, -1 otherwise.
 */
function tryReadTimeLiteral(line, pos) {
    const len = line.length;
    let i = pos;

    for (let component = 0; component < 4; component++) {
        // Each component: 1-2 digits (3+ digits → reject)
        if (i >= len || !(line[i] >= "0" && line[i] <= "9")) return -1;
        i++;
        if (i < len && line[i] >= "0" && line[i] <= "9") {
            i++;
            if (i < len && line[i] >= "0" && line[i] <= "9") return -1; // 3+ digits
        }
        // Colon separator after each component except the last
        if (component < 3) {
            if (i >= len || line[i] !== ":") return -1;
            i++;
        }
    }

    return i;
}

function makeToken(type, value, line, char) {
    return { type, value, line, char };
}

// ---------------------------------------------------------------------------
// Hash mode state tracking
// ---------------------------------------------------------------------------

/**
 * Given the current hash mode and the tokens emitted for the current line,
 * return the hash mode that should apply to the next line.
 *
 * We look backwards through the token array for the first meaningful (non-whitespace,
 * non-newline, non-comment, non-hash-prefix) token on the current line.
 * If the line contained a HASH-ON or HASH-OFF keyword, update accordingly.
 *
 * This is done post-tokenisation so we don't need a forward peek during scanning.
 */
function updatedHashMode(currentMode, tokens) {
    // Walk backwards to find the KEYWORD token on the most recent line.
    // We look for HASH-ON or HASH-OFF keywords that are preceded only by whitespace /
    // hash-prefix on the same line.
    //
    // Skip the trailing NEWLINE that was just appended for the current line — it sits
    // between the line's content tokens and the previous line, so we must step past it
    // before the scan starts.
    let start = tokens.length - 1;
    if (start >= 0 && tokens[start].type === TokenType.NEWLINE) {
        start--;
    }

    for (let i = start; i >= 0; i--) {
        const tok = tokens[i];

        if (tok.type === TokenType.NEWLINE) {
            // Reached the previous line — stop scanning.
            break;
        }

        if (tok.type === TokenType.KEYWORD) {
            const upper = tok.value.toUpperCase();
            if (upper === "HASH-ON") {
                return true;
            }
            if (upper === "HASH-OFF") {
                return false;
            }
            // Any other keyword — hash mode unchanged.
            break;
        }

        // Skip structural tokens that don't affect classification.
        if (
            tok.type === TokenType.WHITESPACE ||
            tok.type === TokenType.HASH_PREFIX ||
            tok.type === TokenType.COMMENT ||
            tok.type === TokenType.OUTPUT_TEXT
        ) {
            continue;
        }

        // Any other token on this line — hash mode unchanged.
        break;
    }

    return currentMode;
}

export { tokenize };
