Files
R8Forge/src/r8asm.cc
T
dsh 1e7acc018e hardware: the 74LS machine gets a period bill of materials, and a mechanics plan
Two things: the TTL build is now the main target (Tang Nano looks too modern), and
the packer learned the format that makes the TTL build simple.

Format: r8forge_tape_pack --eprom writes one byte per instruction with the opcode
in the low three bits.  Then ip is the EPROM address and D2..D0 are the opcode --
no shift register, no 16 bit window, no decoder.  It wastes five bits per
instruction, and a 27C512 is exactly 64K, which is the historical instruction
budget: the 8 bit calculator (51378) and the flagship (43676) both fit.  Both
formats live in the same .r8t trailer now (format byte 1 = packed, 2 = one byte
per instruction), decode dispatches on it, and the test runs both and requires
the same output and step count.

Carrier: 74LS DIPs on a green board, 27C512 in a socket with its quartz window,
6116 SRAM, 74LS299 for the rotate, 6850 ACIA for serial, 74LS47 and four seven
segment displays on the panel, 1 MHz can oscillator.  About 31 to 40 chips, 400
to 700 CNY, a few weeks.  The FPGA stays as a bench-only stepping stone.

hardware/mechanics.md: where mechanics actually pays.  The program must stay
electronic (130 metres of paper tape for the calculator), and 512 relays for the
data ring are out, but the pointer is worth it -- > and < are 71 to 73 percent of
all steps, so a 64 position ratchet or uniselector makes the machine physically
walk the ring.  Also listed: an electromagnetic step counter (10 to 20 Hz, so
only in single step mode, with a clutch for free running), a paper tape punch or
plotter for output, a tape/card reader or plain keypad for input, and a hand crank
as the clock.  The rule that keeps this safe: mechanics only ever sit on sensor
and actuator pins and never in the logic path, and the handshake just stops the
sequencer while something moves -- since nothing in the ISA can observe time,
slower is not different.  A relay-only machine is priced honestly too: 60 to 120
relays, 10 to 50 Hz, hello in 10 to 40 seconds, the calculator in four hours, so
that one is an exhibit.

Also fixed: the synthesised macro fallback file name now carries a per-run token.
When the system temp directory is not writable every r8asm falls back to the same
working directory, so parallel ctest runs could write and import the same file --
which showed up as a random one-test failure in 0.39 seconds and nothing else.
2026-11-20 16:35:02 +08:00

983 lines
34 KiB
C++

#include "r8forge/r8asm.hh"
#include <chrono>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <algorithm>
#include <cstddef>
#include <iostream>
#include <limits>
#include <list>
#include <map>
#include <sstream>
#include <stack>
#include <string>
#include <vector>
#define EMPTY_CHARS (" \t\n\r\f\v")
#define OPERATORS ("+-*/^~\"()&|")
namespace r8asm_strtools {
bool is_integer(const std::string &str) {
return !str.empty() &&
std::ranges::all_of(str, [](char c) { return c >= '0' && c <= '9'; });
}
void trim(std::string &str) { // Clean empty chars in the string.
auto start = str.find_first_not_of(EMPTY_CHARS);
if (!(start ==
std::string::npos)) { // The string isn't filled with empty chars.
auto tmp_end = str.find_last_not_of(EMPTY_CHARS);
str = str.substr(start, tmp_end - start + 1);
} else
str = "";
}
std::string peel(std::string &str) { // Peel the first word from string.
for (size_t i = 1; i < str.size();
i++) { // insert space before and after operators appear
if (std::string(OPERATORS).find(str[i]) != std::string::npos) {
if (str[i - 1] != ' ') {
str = str.substr(0, i) + ' ' + str.substr(i, str.size());
i++;
}
if (str[i + 1] != ' ') {
str = str.substr(0, i + 1) + ' ' + str.substr(i + 1, str.size());
}
}
}
trim(str);
auto end = str.find_first_of(EMPTY_CHARS);
if (end != std::string::npos) {
std::string tmp_str = str.substr(0, end);
str = str.substr(end);
trim(str);
return (tmp_str);
} else {
std::string tmp_str = str;
str = "";
return (tmp_str);
}
}
} // namespace r8asm_strtools
#undef EMPTY_CHARS
#undef OPERATORS
void r8asm::ast::set_cell_size(size_t cell_size) {
this->cell_size = cell_size;
// the names a .r8asm program sees stay as they were: _CELL_SIZE is the
// machine word width in bits (cell_size on the C++ side)
scopes = {{{"_CELL_SIZE", static_cast<general_data>(cell_size)},
// 32-bit cells are the ceiling (the VMs store a cell in uint32_t), and
// `1 << 32` is undefined -- spell the largest width out.
{"_CELL_MAX", static_cast<general_data>(cell_size >= 32 ? 0xFFFFFFFFu
: (1u << cell_size) - 1u)},
{"_ROL_STEP", static_cast<general_data>(cell_size - 1)}}};
}
r8asm::general_data r8asm::ast::expand_node_to_data(const ast_node &node) {
general_data to_return = 0;
switch (node.type) {
case ast_node_t::CONSTANT:
to_return = std::stoi(node.value);
break;
case ast_node_t::PARAMETER:
to_return = scopes.back().at('%' + node.value);
break;
case ast_node_t::VARIABLE: {
bool found = false;
for (auto i = scopes.rbegin(); i != scopes.rend(); i++) {
auto tmp = i->find(node.value);
if (tmp != i->end()) {
to_return = tmp->second;
found = true;
break;
}
}
if (!found && reported.insert(node.value).second)
fail("unknown name '" + node.value + "'");
} break;
case ast_node_t::OPERATOR: {
switch (operator_map.at(node.value)) {
case operator_t::ADD:
to_return = expand_node_to_data(node.children[0]) +
expand_node_to_data(node.children[1]);
break;
case operator_t::SUB:
to_return = expand_node_to_data(node.children[0]) -
expand_node_to_data(node.children[1]);
break;
case operator_t::MUL:
to_return = expand_node_to_data(node.children[0]) *
expand_node_to_data(node.children[1]);
break;
case operator_t::DIV:
to_return = expand_node_to_data(node.children[0]) /
expand_node_to_data(node.children[1]);
break;
case operator_t::XOR:
to_return = expand_node_to_data(node.children[0]) ^
expand_node_to_data(node.children[1]);
break;
case operator_t::AND:
to_return = expand_node_to_data(node.children[0]) &
expand_node_to_data(node.children[1]);
break;
case operator_t::OR:
to_return = expand_node_to_data(node.children[0]) |
expand_node_to_data(node.children[1]);
break;
case operator_t::NOT:
to_return = ~expand_node_to_data(node.children[0]);
break;
default:;
break;
}
break;
};
default:
std::cerr << "ERROR: THIS NODE SHOULD NOT BE EXPANDED TO DATA!!!"
<< std::endl;
break;
}
return to_return;
}
std::string r8asm::ast::expand_node_to_string(const ast_node &node) {
return node.value;
}
#define EMPTY_CODEBLOCK \
{ \
r8asm::ast_node_t::CODEBLOCK, "", {} \
}
// TODO: FILL THE BLANK
#define ROT8_NODE(INS) \
{ \
r8asm::ast_node_t::ROT8, std::string("rot8_") + INS, {} \
}
r8asm::ast_node r8asm::ast::expand_node_to_ins(const ast_node &node) {
ast_node to_return = EMPTY_CODEBLOCK;
switch (node.type) {
case ast_node_t::ROT8: {
/*
ROT8
value: rot8_ins
*/
to_return.children = {node};
switch (rot8_map.at(node.value)) {
case rot8_t::STP:
dataptr++;
break;
case rot8_t::BTP:
dataptr--;
break;
default:
break;
}
} break;
case ast_node_t::REPEAT: {
/*
REPEAT
children[0]: count
children[1]: codeblock
*/
for (data count = expand_node_to_data(node.children[0]); count > 0; count--)
for (const auto &i : expand_node_to_ins(node.children[1]).children)
to_return.children.push_back(i);
} break;
case ast_node_t::FOR: {
/*
FOR
value: loop variable name
children[0]: count
children[1]: codeblock
%for <name> <count> expands the body <count> times with <name> bound to
0, 1, ... count-1. Unlike %rep the body is not identical every round --
the name is a compile time constant, so expressions inside the body can
depend on which round it is (that is the whole point: naming cell +k for
a different k every round).
*/
data count = expand_node_to_data(node.children[0]);
for (data k = 0; k < count; k++) {
/* Copy the innermost scope rather than starting an empty one: inside a
macro body a %for loop still has to see that macro's %1, %2, ... */
std::map<std::string, general_data> loop_scope = scopes.back();
loop_scope.insert_or_assign(node.value, k);
scopes.push_back(loop_scope);
for (const auto &i : expand_node_to_ins(node.children[1]).children)
to_return.children.push_back(i);
scopes.pop_back();
}
} break;
case ast_node_t::MACROCALL: {
/*
MACROCALL
value: macro_name
children: arguments
*/
std::map<std::string, general_data> tmp_scope = {};
for (auto i = 0; i < node.children.size(); i++)
tmp_scope.insert_or_assign('%' + std::to_string(i + 1),
// %1 in macro means children[0]
expand_node_to_data(node.children[i]));
scopes.push_back(tmp_scope);
if (macros.find(node.value) == macros.end())
fail("unknown macro '" + node.value + "'");
else
for (const auto &it : expand_node_to_ins(macros.at(node.value)).children)
to_return.children.push_back(it);
scopes.pop_back();
} break;
case ast_node_t::CODEBLOCK: {
/*
CODEBLOCK
children: ins
*/
for (const auto &i : node.children)
for (const auto &ins : expand_node_to_ins(i).children)
to_return.children.push_back(ins);
} break;
case ast_node_t::BUILTIN: {
/*
BUILTIN
value: type
children: args
*/
if (builtin_map.find(node.value) == builtin_map.end()) {
fail("unknown builtin '." +
node.value.substr(std::string("builtin_").size()) + "'");
return to_return;
}
switch (builtin_map.at(node.value)) {
// WARNING: node.value HAS BEEN USED!!! DO NOT CONSUME IT AGAIN!!!
case builtin_t::ANCHOR: {
/*
ANCHOR
children[0]: var_name
*/
scopes.back().insert_or_assign(expand_node_to_string(node.children[0]),
dataptr);
} break;
case builtin_t::TP: {
/*
TP
children[0]: destination
*/
addr destination = expand_node_to_data(node.children[0]);
if (destination > dataptr)
for (addr i = 0; i < destination - dataptr; i++)
to_return.children.push_back(ROT8_NODE("stp"));
else
for (addr i = 0; i < dataptr - destination; i++)
to_return.children.push_back(ROT8_NODE("btp"));
dataptr = destination;
// stp or btp might be created
// so dataptr must be updated at the same time.
} break;
case builtin_t::XOR: {
/*
XOR
children[0]: argument
WARNING: argument bigger than ((1<<cell_size)-1) will be cut off
e.g.
children[0] -> 5(0b101)
when define cell_size 2, only the last 2 bit will be noticed,
and 5 means 0b01(1) here.
then the node will be expanded like this:
flb
ror
ror
*/
data d = expand_node_to_data(node.children[0]);
for (auto i = 0; i < scopes[0].at("_CELL_SIZE"); i++) {
if (d & 1)
to_return.children.push_back(ROT8_NODE("flb"));
to_return.children.push_back(ROT8_NODE("ror"));
d >>= 1;
}
} break;
case builtin_t::LOOP: {
to_return.children.push_back(ROT8_NODE("stp"));
to_return.children.push_back(ROT8_NODE("biz"));
to_return.children.push_back(ROT8_NODE("btp"));
// >[<
} break;
case builtin_t::ENDLOOP: {
// Consume one '1' from the auxiliary cell and pull the next bit down.
// Without the ror, the rnz could only ever see the bit that flb just
// cleared, so a .loop over 0b111 ran once instead of three times
// (SPEC: "consecutive 1s starting from the LSB").
to_return.children.push_back(ROT8_NODE("stp"));
to_return.children.push_back(ROT8_NODE("flb"));
to_return.children.push_back(ROT8_NODE("ror"));
to_return.children.push_back(ROT8_NODE("rnz"));
to_return.children.push_back(ROT8_NODE("btp"));
// >-+]<
} break;
case builtin_t::VAR: {
/*
VAR
children[0]: var_name
children[1]: var_value
*/
scopes.back().insert_or_assign(expand_node_to_string(node.children[0]),
expand_node_to_data(node.children[1]));
} break;
default:
break;
};
} break;
default:;
break;
}
return to_return;
}
#undef ROT8_NODE
#define ROT8_BC_TO_CHAR(bc) ("><+-[].,"[static_cast<int>(bc)])
std::string r8asm::ast::expand_root() {
auto rot8_ins = expand_node_to_ins(root);
std::string rot8_tape = "";
for (const auto &i : rot8_ins.children)
rot8_tape += ROT8_BC_TO_CHAR(rot8_map.at(i.value));
return rot8_tape;
}
#undef ROT8_BC_TO_CHAR
std::string r8asm::export_rot8() { return main_tree.expand_root(); }
bool r8asm::ast::import_macro(const std::string &macro_name,
const ast_node &node) {
return macros.insert_or_assign(macro_name, node).second;
}
#define POINTS_AT_DIRECTIVE(iter, TYPE) \
(iter->tokens[0].type == token_t::DIRECTIVE && \
directive_keywords.at(iter->tokens[0].value) == directive_t::TYPE)
r8asm::ast_node r8asm::parse(const token_stream &stream) {
ast_node to_return = EMPTY_CODEBLOCK;
token_stream tmp_codeblock_stream = {stream.filename};
/*
ast_node tmp_codeblock_node;
std::vector<ast_node> *destination = &to_return.children;
std::vector<ast_node> *last_destination = nullptr;*/
for (auto it = stream.lines.begin(); it < stream.lines.end(); it++) {
switch (it->tokens[0].type) {
case token_t::DIRECTIVE: {
/*
directive will be tokenized like this:
macro raw_add
0 1
or:
macro raw_add 2 + 4
0 1 2 3 4
(though [2] [3] [4] won't be used)
if it's a rep block:
rep 1 + 1
0 1 2 3
if it's a include line:
include filename
0 1
*/
if (directive_keywords.find(it->tokens[0].value) ==
directive_keywords.end()) {
main_tree.fail("unknown directive '%" + it->tokens[0].value + "'");
break;
}
switch (directive_keywords.at(it->tokens[0].value)) {
// match an entire operation (not a cut for arguments) first
case directive_t::MACRO: {
std::string macro_name = it->tokens[1].value;
tmp_codeblock_stream.lines = {};
depth_t layer = 0;
// now it points at %macro line
for (it += 1; !(POINTS_AT_DIRECTIVE(it, ENDMACRO) && (layer == 0));
it++) {
if (POINTS_AT_DIRECTIVE(it, MACRO))
layer++;
else if (POINTS_AT_DIRECTIVE(it, ENDMACRO))
layer--;
tmp_codeblock_stream.lines.push_back(*it);
}
// now it points at %endmacro line
main_tree.import_macro(macro_name, parse(tmp_codeblock_stream));
} break;
case directive_t::REP: {
// build subtree for argument of rep
token_line buffer = {it->number, {}};
for (auto token_it = it->tokens.begin() + 1; // where argument begins
token_it < it->tokens.end(); token_it++)
buffer.tokens.push_back(*token_it);
/*
REPEAT
children[0]: count
children[1]: codeblock
*/
to_return.children.push_back({r8asm::ast_node_t::REPEAT, "", {}});
to_return.children.back().children.push_back(
parse_single_arg(buffer)); // count will be pushed here.
tmp_codeblock_stream.lines = {};
depth_t layer = 0;
// now it points at %rep line
for (it += 1; !(POINTS_AT_DIRECTIVE(it, ENDREP) && (layer == 0));
it++) {
if (POINTS_AT_DIRECTIVE(it, REP) || POINTS_AT_DIRECTIVE(it, FOR))
layer++;
else if (POINTS_AT_DIRECTIVE(it, ENDREP) ||
POINTS_AT_DIRECTIVE(it, ENDFOR))
layer--;
tmp_codeblock_stream.lines.push_back(*it);
}
// now it points at %endrep line
to_return.children.back().children.push_back(
parse(tmp_codeblock_stream)); // codeblock will be pushed here.
} break;
case directive_t::FOR: {
/*
FOR
value: loop variable name
children[0]: count
children[1]: codeblock
*/
if (it->tokens.size() < 3) {
main_tree.fail("%for needs a name and a count");
break;
}
const std::string loop_name = it->tokens[1].value;
token_line buffer = {it->number, {}};
for (auto token_it = it->tokens.begin() + 2; // name, then the count
token_it < it->tokens.end(); token_it++)
buffer.tokens.push_back(*token_it);
to_return.children.push_back({r8asm::ast_node_t::FOR, loop_name, {}});
to_return.children.back().children.push_back(parse_single_arg(buffer));
tmp_codeblock_stream.lines = {};
depth_t for_layer = 0;
for (it += 1;
!((POINTS_AT_DIRECTIVE(it, ENDFOR) ||
POINTS_AT_DIRECTIVE(it, ENDREP)) &&
(for_layer == 0));
it++) {
if (POINTS_AT_DIRECTIVE(it, FOR) || POINTS_AT_DIRECTIVE(it, REP))
for_layer++;
else if (POINTS_AT_DIRECTIVE(it, ENDFOR) ||
POINTS_AT_DIRECTIVE(it, ENDREP))
for_layer--;
tmp_codeblock_stream.lines.push_back(*it);
}
to_return.children.back().children.push_back(
parse(tmp_codeblock_stream));
} break;
case directive_t::INCLUDE: {
std::string filename = "";
auto tp = it->tokens.begin() + 1;
// now tp points at the LQUOTE
for (tp += 1; tp->type != token_t::RQUOTE; tp++)
filename += tp->value;
// now tp points at RQUOTE
import_file(filename, main_tree.cell_size);
// import_file has already said why if it failed; the error flag is
// set, so this file will not be reported as assembled.
} break;
case directive_t::SYNTH: {
// %synth <name>
// params <n>
// in %1 %2 %3
// out %4 %5
// map <2^|in| numbers>
// %endsynth
const std::string name = it->tokens[1].value;
ast::synth_spec spec;
depth_t layer = 0;
for (it += 1; !(POINTS_AT_DIRECTIVE(it, ENDSYNTH) && layer == 0); it++) {
if (POINTS_AT_DIRECTIVE(it, SYNTH))
layer++;
else if (POINTS_AT_DIRECTIVE(it, ENDSYNTH))
layer--;
if (it->tokens.empty())
continue;
const std::string &kw = it->tokens[0].value;
if (kw == "params") {
spec.params = std::stoi(it->tokens[1].value);
} else if (kw == "inputs" || kw == "outputs") {
// implied by the table now: with map the inputs are %1..%K (K from
// the number of entries) and the outputs follow them in bit order
main_tree.fail("'" + kw +
"' is gone: map already says which parameter is "
"which, use stat for the raw table");
} else if (kw == "metric") {
// it is a property of the run, not of the block: the assembler
// optimises for a layout free block by default, and --synth-chars
// switches every block in the file to the real layout
main_tree.fail("'metric' is gone: blocks are layout free by "
"default, run r8forge --synth-chars to charge the "
"real .tp walk instead");
} else if (kw == "mode") {
const std::string &m = it->tokens[1].value;
if (m == "layers")
spec.mode = 1;
else if (m == "tape")
spec.mode = 0;
else
main_tree.fail("unknown mode '" + m + "'");
} else if (kw == "layout") {
for (size_t i = 1; i < it->tokens.size(); i++)
spec.layout.push_back(std::stoi(it->tokens[i].value));
} else if (kw == "map") {
for (size_t i = 1; i < it->tokens.size(); i++)
spec.map.push_back(std::stoi(it->tokens[i].value));
} else if (kw == "stat") {
// the whole state table: one P-bit value per input combination
for (size_t i = 1; i < it->tokens.size(); i++)
spec.stat.push_back(std::stoi(it->tokens[i].value));
} else {
main_tree.fail("unknown line '" + kw + "' inside %synth");
}
}
// map alone says which parameter is which: 2^K entries mean %1..%K are
// the inputs and the value's bits land on the parameters after them
if (!spec.map.empty() && spec.in.empty()) {
size_t k = 0;
while ((size_t(1) << k) < spec.map.size())
k++;
if ((size_t(1) << k) != spec.map.size()) {
main_tree.fail("%synth '" + name + "': map needs 2^inputs entries");
} else {
for (size_t i = 0; i < k; i++)
spec.in.push_back(int(i));
int width = 0;
for (int val : spec.map)
for (int b = 0; b < 31; b++)
if ((val >> b) & 1)
width = std::max(width, b + 1);
for (int i = 0; i < width; i++)
spec.out.push_back(int(k) + i);
if (int(k) + width > spec.params)
main_tree.fail("%synth '" + name + "': map does not fit in " +
std::to_string(spec.params) + " parameters");
}
}
auto prog = r8asm_synth_program(spec.params, spec.in, spec.out,
spec.map, spec.stat, name, spec.layout,
spec.mode, main_tree.synth_metric,
main_tree.synth_threads, main_tree.quiet);
if (prog.empty()) {
main_tree.fail("%synth '" + name + "': nothing emitted");
} else {
// Write the synthesised body out as a macro and import it: the code
// then goes through the normal, well tested expansion path instead
// of being spliced by hand. set_1 is spelled out (set_0 + flb) so
// the generated file needs nothing from lib.r8asm.
//
// It goes to the system temp directory, and to the working directory
// if that one cannot be written: a sandbox may allow a process to
// write only inside its own tree, and then temp_directory_path()
// happily returns a path whose open() fails.
std::ostringstream body;
body << "%macro " << name << " " << spec.params << "\n";
for (const auto &i : prog) {
body << ".tp %" << i[0] + 1 << "\nbiz\n";
for (int b = 0; b < 8; b++)
if (i[1] & (1 << b)) {
body << ".tp %" << b + 1 << "\n";
if (i[2])
body << "biz\nflb\nrnz\n"; // set_0
body << "flb\n";
}
body << ".tp %" << i[0] + 1 << "\nflb\nrnz\n";
}
body << "%endmacro\n";
std::vector<std::filesystem::path> dirs;
std::error_code ec;
const std::filesystem::path temp_dir =
std::filesystem::temp_directory_path(ec);
if (!ec)
dirs.push_back(temp_dir);
const std::filesystem::path work_dir =
std::filesystem::current_path(ec);
if (!ec)
dirs.push_back(work_dir);
// The file name carries a per-run token: when the system temp directory
// is not writable every process falls back to the *same* working
// directory, and two runs in parallel (ctest -j) would write and import
// the same file -- one of them then assembles the other's body, or
// imports a half written one. That is a race that shows up as a
// random, fast, one-test failure and nothing else. A clock tick is
// enough of a token here and needs no platform include.
const std::string unique =
name + "." +
std::to_string(std::chrono::steady_clock::now()
.time_since_epoch()
.count()) +
".synth.r8asm";
std::string path;
for (size_t d = 0; d < dirs.size(); d++) {
const std::filesystem::path candidate = dirs[d] / unique;
std::ofstream file(candidate, std::ios::binary);
if (!file)
continue;
file << body.str();
file.close();
if (!file)
continue;
if (d > 0 && !main_tree.quiet)
std::cerr << "r8asm: %synth " << name << ": "
<< dirs.front().string()
<< " is not writable, using " << dirs[d].string()
<< std::endl;
path = candidate.string();
break;
}
if (path.empty()) {
main_tree.fail("%synth '" + name +
"': cannot write the synthesised macro anywhere");
} else {
import_file(path, main_tree.cell_size);
std::remove(path.c_str());
}
}
} break;
default:
break;
}
} break;
case token_t::BUILTIN: {
/*
BUILTIN
value: type
children: args
*/
/* like this:
builtin_var pos1 15
0 1 2
*/
to_return.children.push_back(
{r8asm::ast_node_t::BUILTIN, it->tokens[0].value, {}});
// build subtree for argument(s) of builtin
token_line buffer = {it->number, {}};
for (auto token_it = it->tokens.begin() + 1; // where argument begins
token_it < it->tokens.end(); token_it++)
buffer.tokens.push_back(*token_it);
auto args = parse_args(buffer).children;
for (const auto &arg_it : args)
to_return.children.back().children.push_back(
arg_it); // though all string args are treated as VARIABLE, I guess
// builtin_var will read children[0].value successfully.
} break;
case token_t::MACROCALL: {
/* like this:
raw_add 2 4 4 + 2 ( 4 + 4 ) pos1 pos1 + 2
0 1 ...
*/
to_return.children.push_back(
{r8asm::ast_node_t::MACROCALL, it->tokens[0].value, {}});
// build subtree for argument(s) of macro
token_line buffer = {it->number, {}};
for (auto token_it = it->tokens.begin() + 1; // where argument begins
token_it < it->tokens.end(); token_it++)
buffer.tokens.push_back(*token_it);
/*
MACROCALL
value: macro_name
children: arguments
*/
auto args = parse_args(buffer).children;
for (const auto &arg_it : args)
to_return.children.back().children.push_back(arg_it);
} break;
case token_t::ROT8: {
to_return.children.push_back(
{r8asm::ast_node_t::ROT8, it->tokens[0].value, {}});
} break;
default:
break;
}
}
return to_return;
}
#undef POINTS_AT_DIRECTIVE
#define UNRELATED_TO_LEFT(present_token) \
(it != line.tokens.begin() && \
((present_token->type == token_t::LPAREN || \
present_token->type == token_t::STRING || \
present_token->type == token_t::PARAMETER || \
(present_token->type == token_t::OPERATOR && \
(present_token->value == "op_neg" || \
present_token->value == "op_not"))) && \
(!((present_token - 1)->type == token_t::OPERATOR || \
(present_token - 1)->type == token_t::LPAREN))))
r8asm::ast_node r8asm::parse_args(const token_line &line) {
ast_node to_return = EMPTY_CODEBLOCK;
// arguments will be storaged in to_return.children
std::vector<token_line> raw_args;
std::list<token> buffer;
for (auto it = line.tokens.begin(); it != line.tokens.end(); it++) {
if (UNRELATED_TO_LEFT(it)) {
raw_args.push_back({line.number, {}});
while (!buffer.empty()) {
raw_args.back().tokens.push_back(buffer.front());
buffer.pop_front();
}
}
buffer.push_back(*it);
}
if (!buffer.empty()) {
raw_args.push_back({line.number, {}});
while (!buffer.empty()) {
raw_args.back().tokens.push_back(buffer.front());
buffer.pop_front();
}
}
for (const auto &it : raw_args)
to_return.children.push_back(parse_single_arg(it));
return to_return;
}
#undef UNRELATED_TO_LEFT
r8asm::ast_node r8asm::parse_single_arg(const token_line &line) {
ast_node to_return;
std::stack<ast_node> data_stack;
std::stack<ast_node> symbol_stack;
depth_t right_assoc_depth = 0;
auto deal_with_assoc = [&data_stack, &symbol_stack, &right_assoc_depth] {
while (right_assoc_depth) {
auto unary_op = symbol_stack.top();
symbol_stack.pop();
unary_op.children.push_back(data_stack.top());
data_stack.pop();
data_stack.push(unary_op);
right_assoc_depth--;
}
};
for (auto it = line.tokens.begin(); it < line.tokens.end(); it++) {
switch (it->type) {
case token_t::LPAREN: {
depth_t layer = 0;
token_line buffer = {line.number, {}};
for (it += 1; !((it->type == token_t::RPAREN) && (layer == 0)); it++) {
if (it->type == token_t::LPAREN)
layer++;
else if (it->type == token_t::RPAREN)
layer--;
buffer.tokens.push_back(*it);
}
data_stack.push(parse_single_arg(buffer));
deal_with_assoc();
} break;
case token_t::STRING: {
{
if (r8asm_strtools::is_integer(it->value))
data_stack.push({r8asm::ast_node_t::CONSTANT, it->value, {}});
else
data_stack.push({r8asm::ast_node_t::VARIABLE, it->value, {}});
}
deal_with_assoc();
} break;
case token_t::PARAMETER: {
data_stack.push({r8asm::ast_node_t::PARAMETER, (it->value), {}});
deal_with_assoc();
} break;
case token_t::OPERATOR:
// precedence: NEG = NOT
// > MUL = DIV
// > ADD = SUB
// > AND
// > XOR
// > OR
{
#define GET_PRECEDENCE(op_key) \
(main_tree.op_precedence_map.at(main_tree.operator_map.at(op_key)))
switch (main_tree.operator_map.at(it->value)) {
case ast::operator_t::NOT: // unary operators
{
right_assoc_depth++;
symbol_stack.push({ast_node_t::OPERATOR, it->value, {}});
} break;
default: // now all of them are binary operators
{
while ((!symbol_stack.empty()) &&
GET_PRECEDENCE(it->value) >=
GET_PRECEDENCE(symbol_stack.top().value))
// e.g. pushing + while here's already a * in symbol_stack
{
auto tmp_op = symbol_stack.top();
symbol_stack.pop();
auto tmp_arg = data_stack.top(); // 2nd arg got
data_stack.pop();
tmp_op.children.push_back(data_stack.top()); // 1st arg pushed
data_stack.pop();
tmp_op.children.push_back(tmp_arg);
data_stack.push(tmp_op);
}
symbol_stack.push({ast_node_t::OPERATOR, it->value, {}});
} break;
}
}
#undef GET_PRECEDENCE
default:
break;
}
}
while (!symbol_stack.empty()) {
auto tmp_op = symbol_stack.top();
symbol_stack.pop();
auto tmp_arg = data_stack.top(); // 2nd arg got
data_stack.pop();
tmp_op.children.push_back(data_stack.top()); // 1st arg pushed
data_stack.pop();
tmp_op.children.push_back(tmp_arg);
data_stack.push(tmp_op);
}
to_return = data_stack.top();
return to_return;
}
#undef EMPTY_CODEBLOCK
#define OPERATORS ("+-*/^~&|")
std::vector<r8asm::token_line>
r8asm::token_stream::lex(const std::string &char_stream) {
std::vector<token_line> to_return;
std::istringstream iss(char_stream);
std::string str_buffer;
line_number num = 0;
bool next_will_be_remark = false;
const std::map<std::string, std::string> operator_map = {
// without "-"
{"+", "op_add"}, {"*", "op_mul"}, {"/", "op_div"}, {"^", "op_xor"},
{"&", "op_and"}, {"|", "op_or"}, {"~", "op_not"}, {"-", "op_sub"},
};
bool first_line = true;
while (std::getline(iss, str_buffer)) {
// a UTF-8 BOM would otherwise end up glued to the first token and every
// editor on Windows writes one
if (first_line && str_buffer.size() >= 3 &&
(unsigned char)str_buffer[0] == 0xEF &&
(unsigned char)str_buffer[1] == 0xBB &&
(unsigned char)str_buffer[2] == 0xBF)
str_buffer.erase(0, 3);
first_line = false;
std::vector<token> token_buffer = {};
bool start_of_line = true;
bool lquote_appeared = false;
for (std::string token_val = r8asm_strtools::peel(str_buffer);
token_val != ""; token_val = r8asm_strtools::peel(str_buffer)) {
if (lquote_appeared && token_val[0] != '\"')
token_buffer.push_back({r8asm::token_t::STRING, token_val});
else {
if (start_of_line) { // it's the first token in the line
start_of_line = false;
if (token_val[0] == '.') // builtin
token_buffer.push_back(
{r8asm::token_t::BUILTIN, "builtin_" + token_val.substr(1)});
// e.g. .var -> builtin_var
else if (token_val[0] == '%') // directive
token_buffer.push_back(
{r8asm::token_t::DIRECTIVE, token_val.substr(1)});
// e.g. %macro -> macro
else if (token_val[0] == ';') // remark
next_will_be_remark = true;
else if (raw_rot8_ins.find(token_val) != raw_rot8_ins.end()) // rot8
token_buffer.push_back({r8asm::token_t::ROT8, "rot8_" + token_val});
else // macro
token_buffer.push_back({r8asm::token_t::MACROCALL, token_val});
} else { // it's a part of argument
if (!next_will_be_remark) {
if (token_val[0] == ';') // remark
next_will_be_remark = true;
else if (token_val[0] == '%') // parameter
token_buffer.push_back(
{r8asm::token_t::PARAMETER, token_val.substr(1)});
else if (token_val[0] == '"') { // quote(filename)
if (!lquote_appeared) {
lquote_appeared = true; // only %include will use quotes, they
// won't appear again
token_buffer.push_back({r8asm::token_t::LQUOTE, token_val});
} else
token_buffer.push_back({r8asm::token_t::RQUOTE, token_val});
} else if (token_val[0] == '(')
token_buffer.push_back({r8asm::token_t::LPAREN, token_val});
else if (token_val[0] == ')')
token_buffer.push_back({r8asm::token_t::RPAREN, token_val});
else if (std::string(OPERATORS).find(token_val[0]) !=
std::string::npos) // operator
token_buffer.push_back(
{r8asm::token_t::OPERATOR, operator_map.at(token_val)});
else
token_buffer.push_back({r8asm::token_t::STRING, token_val});
}
}
}
}
num++;
if (!token_buffer.empty())
to_return.push_back({num, token_buffer});
next_will_be_remark = false;
}
return to_return;
}
#undef OPERATORS
bool r8asm::import_file(const std::string &filename, size_t cell_size) {
token_stream stream(filename);
if (!stream.ok) {
main_tree.fail("cannot open '" + filename + "'");
return (false);
}
// Before parsing: %include asks main_tree.cell_size for the width to hand
// to the included file, and that used to answer 0 because the size was
// only set at the end of this function.
main_tree.set_cell_size(cell_size);
for (const auto &token_it : parse(stream).children) {
main_tree.root.children.push_back(token_it);
}
return (!main_tree.error);
}