Boost Spirit 将表达式标记化为向量

Question

我正在尝试解析一个也可以包含标识符的表达式并将每个元素推送到 std::vector <std::string> 中，我想出了以下语法:

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <vector>

namespace qi = boost::spirit::qi;

struct Tokeniser
    : boost::spirit::qi::grammar <std::string::const_iterator, std::vector <std::string> (), boost::spirit::ascii::space_type>
{
    Tokeniser() : Tokeniser::base_type(expression)
    {
        namespace qi = boost::spirit::qi;
        expression = 
            term >>
            *( (qi::string("+")[qi::_val.push_back(qi::_1)] >> term) |
               (qi::string("-")[qi::_val.push_back(qi::_1)] >> term) );

        term = 
            factor >>
            *( (qi::string("*")[qi::_val.push_back(qi::_1)] >> factor) |
               (qi::string("/")[qi::_val.push_back(qi::_1)] >> factor) );

        factor =
            (identifier | myDouble_)[qi::_val.push_back(qi::_1)] |
            qi::string("(")[qi::_val.push_back(qi::_1)] >> expression >> qi::string(")")[qi::_val.push_back(qi::_1)];

        identifier = qi::raw [ qi::lexeme[ (qi::alpha | '_') >> *(qi::alnum | '_') ] ]; 

        myDouble_ = qi::raw [ qi::double_ ];
    }

    boost::spirit::qi::rule<std::string::const_iterator, std::vector <std::string> (), boost::spirit::ascii::space_type> expression;
    boost::spirit::qi::rule<std::string::const_iterator, boost::spirit::ascii::space_type>                               factor;
    boost::spirit::qi::rule<std::string::const_iterator, boost::spirit::ascii::space_type>                               term;        

    boost::spirit::qi::rule<std::string::const_iterator, std::string(), boost::spirit::ascii::space_type> identifier;
    boost::spirit::qi::rule<std::string::const_iterator, std::string(), boost::spirit::ascii::space_type> myDouble_;
};

但是，我收到以下错误 'const struct boost::phoenix::actor<boost::spirit::attribute<0> >' has no member named 'push_back' .

是否有直接的方法来执行我想做的事情？

Answer 1

是的，占位符类型(显然)没有 push_back 成员。

C++ 是强类型的。任何延迟的 Action 都是一种“幻觉”: Actor 通过组合可以稍后“评估”的特殊用途类型在表达式模板中表示。

表达式模板简介

Live On Coliru

为了以防万一您想了解它实际上 是如何工作的，请从头开始一个简单的示例。注释描述了代码各个部分的作用:

// we have lazy placeholder types:
template <int N> struct placeholder {};
placeholder<1> _1;
placeholder<2> _2;
placeholder<3> _3;

// note that every type here is stateless, and acts just like a more
// complicated placeholder.
// We can have expressions, like binary addition:
template <typename L, typename R> struct addition { };
template <typename L, typename R> struct multiplication { };

// here is the "factory" for our expression template:
template <typename L, typename R> addition<L,R> operator+(L const&, R const&) { return {}; }
template <typename L, typename R> multiplication<L,R> operator*(L const&, R const&) { return {}; }

///////////////////////////////////////////////
// To evaluate/interpret the expressions, we have to define "evaluation" for each type of placeholder:
template <typename Ctx, int N> 
    auto eval(Ctx& ctx, placeholder<N>) { return ctx.arg(N); }
template <typename Ctx, typename L, typename R>
    auto eval(Ctx& ctx, addition<L, R>) { return eval(ctx, L{}) + eval(ctx, R{}); }
template <typename Ctx, typename L, typename R>
    auto eval(Ctx& ctx, multiplication<L, R>) { return eval(ctx, L{}) * eval(ctx, R{}); }

///////////////////////////////////////////////
// A simple real-life context would contain the arguments:
#include <vector>
struct Context {
    std::vector<double> _args;

    // define the operation to get an argument from this context:
    double arg(int i) const { return _args.at(i-1); }
};

#include <iostream>
int main() {
    auto foo = _1 + _2 + _3;

    Context ctx { { 3, 10, -4 } };
    std::cout << "foo: " << eval(ctx, foo) << "\n";
    std::cout << "_1 + _2 * _3: " << eval(ctx, _1 + _2 * _3) << "\n";
}

输出正是您所期望的:

foo: 9
_1 + _2 * _3: -37

修复 Action :

您必须“描述”push_back 操作，而不是尝试在占位符上查找此类操作。凤凰有你的支持:

#include <boost/phoenix/stl.hpp>

现在我将简化使用 phoenix::push_back 的操作:

auto push = px::push_back(qi::_val, qi::_1);

expression = 
    term >>
    *( (qi::string("+")[push] >> term) |
       (qi::string("-")[push] >> term) );

term = 
    factor >>
    *( (qi::string("*")[push] >> factor) |
       (qi::string("/")[push] >> factor) );

factor =
    (identifier | myDouble_)[push] |
    qi::string("(")[push] >> expression >> qi::string(")")[push];

// etc.

但是，这有一个额外的问题，即 _val 被解析为规则的属性类型。但是你的一些规则没有声明属性类型，所以它默认为qi::unused_type。显然，为该属性生成“push_back”评估代码不适用于 unused_type。

让我们修复这些声明:

qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression;
qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> factor;
qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> term;

其他问题!

当我们这样做时， token 基本上是空的。给了什么？

在存在语义 Action 的情况下，自动属性传播被禁止。因此，您必须努力获取附加到最终标记向量的子表达式的内容。

同样，使用 Phoenix 的 STL 支持:

auto push      = px::push_back(qi::_val, qi::_1);
auto propagate = px::insert(qi::_val, px::end(qi::_val), px::begin(qi::_1), px::end(qi::_1));

expression = 
    term[propagate] >>
    *( (qi::string("+")[push] >> term[propagate]) |
       (qi::string("-")[push] >> term[propagate]) );

term = 
    factor[propagate] >>
    *( (qi::string("*")[push] >> factor[propagate]) |
       (qi::string("/")[push] >> factor[propagate]) );

factor =
    (identifier | myDouble_)[push] |
    qi::string("(")[push] >> expression[propagate] >> qi::string(")")[push];

现在，使用 Live On Coliru 进行测试

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/phoenix/stl.hpp>
#include <vector>

namespace qi = boost::spirit::qi;
namespace px = boost::phoenix;

template <typename It = std::string::const_iterator>
struct Tokeniser
    : qi::grammar <It, std::vector <std::string> (), boost::spirit::ascii::space_type>
{
    Tokeniser() : Tokeniser::base_type(expression)
    {
        auto push      = px::push_back(qi::_val, qi::_1);
        auto propagate = px::insert(qi::_val, px::end(qi::_val), px::begin(qi::_1), px::end(qi::_1));

        expression = 
            term[propagate] >>
            *( (qi::string("+")[push] >> term[propagate]) |
               (qi::string("-")[push] >> term[propagate]) );

        term = 
            factor[propagate] >>
            *( (qi::string("*")[push] >> factor[propagate]) |
               (qi::string("/")[push] >> factor[propagate]) );

        factor =
            (identifier | myDouble_)[push] |
            qi::string("(")[push] >> expression[propagate] >> qi::string(")")[push];

        identifier = qi::raw [ qi::lexeme[ (qi::alpha | '_') >> *(qi::alnum | '_') ] ]; 

        myDouble_ = qi::raw [ qi::double_ ];

        BOOST_SPIRIT_DEBUG_NODES((expression)(term)(factor)(identifier)(myDouble_))
    }

    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> factor;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> term;
    qi::rule<It, std::string(), boost::spirit::ascii::space_type> identifier;
    qi::rule<It, std::string(), boost::spirit::ascii::space_type> myDouble_;
};

int main() {
    Tokeniser<> tok;

    std::string const input = "x + 89/(y*y)";

    auto f = input.begin(), l = input.end();
    std::vector<std::string> tokens;
    if (phrase_parse(f, l, tok, boost::spirit::ascii::space, tokens)) {
        std::cout << "Parsed " << tokens.size() << " tokens:\n";
        for (auto& token : tokens)
            std::cout << " - '" << token << "'\n";
    } else {
        std::cout << "Parse failed\n";
    }

    if (f != l)
        std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}

打印

Parsed 9 tokens:
 - 'x'
 - '+'
 - '89'
 - '/'
 - '('
 - 'y'
 - '*'
 - 'y'
 - ')'

大清理

一般来说，避免语义操作(参见我的回答 Boost Spirit: "Semantic actions are evil"? - 特别是关于副作用的项目符号)。大多数时候，您可以摆脱自动属性传播。我想说这是 Boost Spirit 的主要卖点。

进一步简化 skipping/lexemes ( Boost spirit skipper issues ) 确实显着减少了代码和编译时间:

Live On Coliru

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <vector>

namespace qi = boost::spirit::qi;

template <typename It = std::string::const_iterator>
struct Tokeniser : qi::grammar <It, std::vector <std::string>()> {
    Tokeniser() : Tokeniser::base_type(start)
    {
        start = qi::skip(boost::spirit::ascii::space) [expression];
        expression =
            term >>
            *( (qi::string("+") >> term) |
               (qi::string("-") >> term) );

        term =
            factor >>
            *( (qi::string("*") >> factor) |
               (qi::string("/") >> factor) );

        factor =
            (identifier | myDouble_) |
            qi::string("(") >> expression >> qi::string(")");

        identifier = qi::raw [ (qi::alpha | '_') >> *(qi::alnum | '_') ]; 
        myDouble_  = qi::raw [ qi::double_ ];

        BOOST_SPIRIT_DEBUG_NODES((expression)(term)(factor)(identifier)(myDouble_))
    }

    qi::rule<It, std::vector<std::string>()> start;
    qi::rule<It, std::vector<std::string>(), boost::spirit::ascii::space_type> expression, factor, term;
    qi::rule<It, std::string()> identifier, myDouble_;
};

int main() {
    Tokeniser<> tok;

    std::string const input = "x + 89/(y*y)";

    auto f = input.begin(), l = input.end();
    std::vector<std::string> tokens;
    if (parse(f, l, tok, tokens)) {
        std::cout << "Parsed " << tokens.size() << " tokens:\n";
        for (auto& token : tokens)
            std::cout << " - '" << token << "'\n";
    } else {
        std::cout << "Parse failed\n";
    }

    if (f != l)
        std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}

静态打印

Parsed 9 tokens:
 - 'x'
 - '+'
 - '89'
 - '/'
 - '('
 - 'y'
 - '*'
 - 'y'
 - ')'

遗留问题

您是否考虑过回溯行为？我认为您需要在规则中明智地放置一些 qi::hold[] 指令，例如，参见Understanding Boost.spirit's string parser

大问题:

1. 当您想要的只是标记时，为什么还要“解析”？
2. 你会用这些代币做什么？解析器构建了很多信息，只是为了再次丢弃它