grammar.g

header {
	//don't try to add all the headers inside our namespace
	ANTLR_END_NAMESPACE

	#include <libwccl/parser/ParserException.h>

	#include <cstdio>
	#include <antlr/Token.hpp>
	#include <boost/lexical_cast.hpp>

	// values/variables
	#include <libwccl/variables.h>
	#include <libwccl/values/bool.h>
	#include <libwccl/values/tset.h>
	#include <libwccl/values/strset.h>
	#include <libwccl/values/position.h>
	
	// sentence context
	#include <libwccl/sentencecontext.h>

	// operators
	#include <libwccl/ops/operator.h>

	#include <libwccl/ops/functions/constant.h>
	#include <libwccl/ops/functions/vargetter.h>
	#include <libwccl/ops/functions/conditional.h>

	#include <libwccl/ops/functions/bool/varsetter.h>
	#include <libwccl/ops/functions/bool/predicates/debug.h>
	#include <libwccl/ops/functions/bool/predicates/or.h>
	#include <libwccl/ops/functions/bool/predicates/nor.h>
	#include <libwccl/ops/functions/bool/predicates/and.h>
	#include <libwccl/ops/functions/bool/predicates/regex.h>
	#include <libwccl/ops/functions/bool/predicates/intersects.h>
	#include <libwccl/ops/functions/bool/predicates/issubsetof.h>
	#include <libwccl/ops/functions/bool/predicates/isinside.h>
	#include <libwccl/ops/functions/bool/predicates/isoutside.h>
	#include <libwccl/ops/functions/bool/predicates/equals.h>
	#include <libwccl/ops/functions/bool/predicates/weakagreement.h>
	#include <libwccl/ops/functions/bool/predicates/pointagreement.h>
	#include <libwccl/ops/functions/bool/predicates/strongagreement.h>

	#include <libwccl/ops/functions/strset/affix.h>
	#include <libwccl/ops/functions/strset/getorth.h>
	#include <libwccl/ops/functions/strset/toupper.h>
	#include <libwccl/ops/functions/strset/tolower.h>
	#include <libwccl/ops/functions/strset/getlemmas.h>
	
	#include <libwccl/ops/functions/tset/agrfilter.h>
	#include <libwccl/ops/functions/tset/catfilter.h>
	#include <libwccl/ops/functions/tset/getsymbols.h>
	#include <libwccl/ops/functions/tset/getwordclass.h>
	#include <libwccl/ops/functions/tset/getsymbolsinrange.h>

	#include <libwccl/ops/functions/position/relativeposition.h>

	#include <libwccl/ops/functions/bool/iterations/only.h>
	#include <libwccl/ops/functions/bool/iterations/atleast.h>
	#include <libwccl/ops/functions/bool/iterations/leftlook.h>
	#include <libwccl/ops/functions/bool/iterations/rightlook.h>
	
	// Rules, actions
	#include <libwccl/ops/tagrule.h>
	#include <libwccl/ops/rulesequence.h>
	//
	#include <libwccl/ops/tagactions/unify.h>
	#include <libwccl/ops/tagactions/delete.h>
	#include <libwccl/ops/tagactions/select.h>
	#include <libwccl/ops/tagactions/relabel.h>
	#include <libwccl/ops/tagactions/mark.h>
	#include <libwccl/ops/tagactions/unmark.h>

	// Match operators
	#include <libwccl/values/tokenmatch.h>
	#include <libwccl/values/annotationmatch.h>
	#include <libwccl/values/matchvector.h>
	#include <libwccl/ops/match/applyoperator.h>
	#include <libwccl/ops/match/conditions/optionalmatch.h>
	#include <libwccl/ops/match/conditions/repeatedmatch.h>
	#include <libwccl/ops/match/conditions/conjconditions.h>

	// Unicode String
	#include <unicode/uniset.h>
	#include <unicode/unistr.h>

	// start our namespace again
	ANTLR_BEGIN_NAMESPACE(Wccl)
}

options {
	language = "Cpp";
	genHashLines = false;
	namespace = "Wccl";
//	genHashLines = true;
}

// ----------------------------------------------------------------------------
// ANTLR PARSER
// ----------------------------------------------------------------------------
class ANTLRParser extends Parser;
options {
	k = 1;
	buildAST = false;
	exportVocab = ANTLRExpr;
	defaultErrorHandler = false;
}
{
private:
	// 
	const UnicodeString token_ref_to_ustring(antlr::RefToken& rstr) const { 
		return UnicodeString::fromUTF8(((antlr::Token*)rstr)->getText().c_str()).unescape();
	}
	/*
	const UnicodeString str_token_ref_to_ustring(antlr::RefToken& rstr) const { 
		UnicodeString ret_ustr, ustr = token_ref_to_ustring(rstr);

		if (ustr.length() < 3) {
			return "";
		}

		ustr.extract(1, ustr.length() - 2, ret_ustr);

		return ret_ustr;
	}
	*/
	//
	const std::string str_token_rem_grav(antlr::RefToken& rstr) const {
		size_t len = 0;
		std::string ret = token_ref_to_std_string(rstr);

		if ((len = ret.length()) < 2) {
			return ret;
		}

		if (ret[0] == '`' && ret[len - 1] == '`') {
			return ret.substr(1, len - 2);
		}

		return ret;
	}
	//
	const std::string token_ref_to_std_string(antlr::RefToken& rstr) const { 
		return (((antlr::Token*)rstr)->getText());
	}
	//
	int token_ref_to_int(antlr::RefToken& rstr) { 
		return atoi(((antlr::Token*)rstr)->getText().c_str());
	}
}

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// "GLOBAL" RULES
///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////

// ----------------------------------------------------------------------------
// Rule for parsing string set operator with scope. 
// Returns boost::shared_ptr<Operator<StrSet> >
parse_strset_operator
	[const Corpus2::Tagset &tagset]
	returns [boost::shared_ptr<Operator<StrSet> > res]
{
	Variables vars;
	boost::shared_ptr<Function<StrSet> > body;
}
	: body = strset_operator [tagset, vars] {
			res.reset(new Operator<StrSet>(body, vars));
	}
	EOF
;

// ----------------------------------------------------------------------------
// Rule for parsing bool operator with scope. 
// Returns boost::shared_ptr<Operator<Bool> > 
parse_bool_operator 
	[const Corpus2::Tagset &tagset]
	returns [boost::shared_ptr<Operator<Bool> > res]
{
	Variables vars;
	boost::shared_ptr<Function<Bool> > body;
}
	: body = bool_operator [tagset, vars] {
			res.reset(new Operator<Bool>(body, vars));
	}
	EOF
;

// ----------------------------------------------------------------------------
// Rule for parsing symbol set operator with scope.
// Returns boost::shared_ptr<Operator<TSet> >
parse_symset_operator
	[const Corpus2::Tagset &tagset]
	returns [boost::shared_ptr<Operator<TSet> > res]
{
	Variables vars;
	boost::shared_ptr<Function<TSet> > body;
}
	: body = symset_operator [tagset, vars] {
			res.reset(new Operator<TSet>(body, vars));
	}
	EOF
;

// ----------------------------------------------------------------------------
// Rule for parsing position operator with scope.
// Returns boost::shared_ptr<Operator<Position> >
parse_position_operator
	[const Corpus2::Tagset &tagset]
	returns [boost::shared_ptr<Operator<Position> > res]
{	
	Variables vars;
	boost::shared_ptr<Function<Position> > body;
}
	: body = position_operator [tagset, vars] {
			res.reset(new Operator<Position>(body, vars));
	}
	EOF
;

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------
// Rule for parsing single WCCL Rule
// Returns boost::shared_ptr<TagRule>
parse_single_rule
	[const Corpus2::Tagset &tagset]
	returns [boost::shared_ptr<TagRule> rle]
{	
	Variables vars;
}
	: rle = rule [tagset, vars]
;

// Rule for parsing rules section in the wccl file
// Returns boost::shared_ptr<RuleSequence>
parse_rule_sequence
	[const Corpus2::Tagset& tagset]
	returns [boost::shared_ptr<RuleSequence> rule_seq]
{
	Variables vars;
}
	: rule_seq = rules[tagset, vars]
;

// ----------------------------------------------------------------------------
// ----------------------------------------------------------------------------
// Rule for parsing the match rules
// Returns boost::shared_ptr<Expression>
parse_match_rule
	[const Corpus2::Tagset& tagset]
	returns [boost::shared_ptr<Expression> ret_match]
{
	Variables vars;
	vars.get_put<Match>("_M");
}
	: ret_match = match_rule_operator[tagset, vars] 
;

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// VALUES
///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////

// ----------------------------------------------------------------------------
// Single or muliple (comma separated) elements in string set, may be:
//   'a' "a" [] ['a'] ['a', 'b'] ["a"] ["a", "b"] ['a', "b"]
// Parsing strset literal and returning plain strset value.
// Returns boost::shared_ptr<StrSet> 
strset_literal 
	returns [boost::shared_ptr<StrSet> s_set]
{
	s_set.reset(new StrSet());
}
	: s0: STRING { 
			s_set->insert(token_ref_to_ustring(s0)); 
		}
	| LBRACKET 
		(
			s1: STRING { 
				s_set->insert(token_ref_to_ustring(s1)); 
			}
	  	(
				COMMA s2: STRING { 
					s_set->insert(token_ref_to_ustring(s2)); 
				}
			)*
	  )? 
		RBRACKET
;
// String set value as constrant string set:
// Returns boost::shared_ptr<Constant<StrSet> >
strset_value 
	returns [boost::shared_ptr<Constant<StrSet> > val]
{
	boost::shared_ptr<StrSet> set;
}
	: set = strset_literal {
		val.reset(new Constant<StrSet>(*set));
	}
;

// ----------------------------------------------------------------------------
// Element of sym set. This rule, inserts element into symbol set 
// with corresponding tagset. 
// WARNING! This rule can throw ParserException! Be careful!
symset_elem
	[const Corpus2::Tagset& tagset, boost::shared_ptr<TSet>& t_set]
	: s1: SYMBOL {
		try {
			t_set->insert_symbol(tagset, str_token_rem_grav(s1));
		}
		catch(Corpus2::TagParseError &e) {
			throw(ParserException(e.info()));
		}
	}
;

// Symset literal. Symset element may be: 
//  a, `a ` (this is guaranteed by lexer rule - SYMBOL) or {a} {`a`} {a, b} 
//  {`a`, `b`} {a, `b`} {`a`, b}
// Parsing symset literal and returning plain symset value.
// Returns boost::shared_ptr<TSet>
symset_literal
	[const Corpus2::Tagset& tagset]
	returns [boost::shared_ptr<TSet> t_set]
{
	t_set.reset(new TSet());
}
	: symset_elem [tagset, t_set]
	| LCURLY 
		(
			symset_elem [tagset, t_set] (COMMA symset_elem [tagset, t_set])* 
		)?
	  RCURLY
;
// Symset value, as constant symbol set
// Returns boost::shared_ptr<Constant<TSet> >
symset_value 
	[const Corpus2::Tagset& tagset]
	returns [boost::shared_ptr<Constant<TSet> > val]
{
	boost::shared_ptr<TSet> set;
}
	: set = symset_literal [tagset] {
		val.reset(new Constant<TSet>(*set));
	}
;

// ----------------------------------------------------------------------------
// Bool literal. May be True or False. Parsing bool literal and returning 
// plain bool value.
// Returns boost::shared_ptr<Bool>
bool_literal
	returns [boost::shared_ptr<Bool> val]
	: "True"  { val.reset(new Bool(Bool(true ))); }
	| "False" { val.reset(new Bool(Bool(false))); }
;
// Bool value, as constat bool Value
// Returns boost::shared_ptr<Constant<Bool> >
bool_value 
	returns [boost::shared_ptr<Constant<Bool> > val]
{
	boost::shared_ptr<Bool> bool_lit;
}
	: bool_lit = bool_literal {
		val.reset(new Constant<Bool>(*bool_lit));
	}
;

// ----------------------------------------------------------------------------
// Position literal may be:
// 	(+|-)?(0-9)+ or  begin or end or nowhere
// Parsing position literal and returning plain position value.
// returns boost::shared_ptr<Position>
position_literal
	returns [boost::shared_ptr<Position> val]
{
	int i = 0;
}
	: i = number {
		val.reset(new Position(Position(i)));
	}
	| "begin" {
		val.reset(new Position(Position(Position::Begin)));
	}
	| "end" {
		val.reset(new Position(Position(Position::End)));
	}
	| "nowhere" {
		val.reset(new Position(Position(Position::Nowhere)));
	}
;

// Position as constant position value
// Returns boost::shared_ptr<Constant<Position> >
position_value
	returns [boost::shared_ptr<Constant<Position> > val]
{
	boost::shared_ptr<Position> pos_lit;
}
	: pos_lit = position_literal {
		val.reset(new Constant<Position>(*pos_lit));
	}
;

// ----------------------------------------------------------------------------
// Value used into match operator such as TOK[position] and ANN[position, name]
// Returns boost::shared_ptr<Match>
match_value
	returns [boost::shared_ptr<Match> val]
{
	boost::shared_ptr<MatchData> m;
}
	: m = match_data_value {
		val.reset(new Match(m));
	}
;

// ----------------------------------------------------------------------------
// Value used into match operator such as TOK[position] and ANN[position, name]
// Returns boost::shared_ptr<MatchData>
match_data_value
	returns [boost::shared_ptr<MatchData> val]
	: val = token_match_value
	| val = ann_match_value
	| val = match_vector_value
;

// token match value
// Returns boost::shared_ptr<TokenMatch>
token_match_value
	returns [boost::shared_ptr<TokenMatch> val]
{
	boost::shared_ptr<Position> p;
}
	: "TOK"LBRACKET p = position_literal RBRACKET {
		val.reset(new TokenMatch(*p));
	}
;

// annotation match value
// Returns boost::shared_ptr<AnnotationMatch> 
ann_match_value
	returns [boost::shared_ptr<AnnotationMatch> val]
{
	boost::shared_ptr<Position> p;
}
	: "ANN" LBRACKET p = position_literal COMMA channel : STRING RBRACKET {
		val.reset(new AnnotationMatch(*p, token_ref_to_std_string(channel)));
	}
;

// annotation match vector: MATCH() or MATCH(token, ann, MATCH())
// Returns boost::shared_ptr<MatchVector>
match_vector_value
	returns [boost::shared_ptr<MatchVector> val]
{
	val.reset(new MatchVector());
}
	: "MATCH" LPAREN (match_vector_value_item[val])? RPAREN
;

// Body of the MATCH value. It only adds vector items to the MatchVector
// Item may be single or multiple
match_vector_value_item [boost::shared_ptr<MatchVector>& mvector]
{
	boost::shared_ptr<Match> m_val;
}
	: m_val = match_value {
		mvector->append(m_val);
	} 
	(
		COMMA
		m_val = match_value {
			mvector->append(m_val);
		}
	)*
;

// ----------------------------------------------------------------------------
// Number may be unsigned or signed: 1, +1, -1
number 
	returns [int ret]
{
	ret = 0;
}
	: s: SIGNED_INT   { ret = token_ref_to_int(s); }
	| u: UNSIGNED_INT { ret = token_ref_to_int(u); }
;

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// VARIABLES
///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////

// ----------------------------------------------------------------------------
// Position: $Name
// Get position variable (however, before put into) from variables
// Returns boost::shared_ptr<VariableAccessor<Position> > 
position_variable_acc
	[Variables& vars] 
	returns [boost::shared_ptr<VariableAccessor<Position> > pos_acc]
	: POS_PREFIX n: SYMBOL { 
		vars.get_put<Position>(str_token_rem_grav(n));

		VariableAccessor<Position> acc = 
			vars.create_accessor<Position>(str_token_rem_grav(n));

		pos_acc.reset(new VariableAccessor<Position>(acc));
	}
;

// VarGetter for Position variable. This rule wrapped position_variable_acc.
// Returs boost::shared_ptr<VarGetter<Position> >
position_variable
	[Variables& vars] 
	returns [boost::shared_ptr<VarGetter<Position> > op]
{
	boost::shared_ptr<VariableAccessor<Position> > pos_acc;
}
	: pos_acc = position_variable_acc [vars] {
		op.reset(new VarGetter<Position>(*pos_acc));
	}
;

// ----------------------------------------------------------------------------
// String set, $s:name
// This expression gets (however, before put into) variable of the type StrSet 
// from scope -- variables.
// Returns boost::shared_ptr<VariableAccessor<StrSet> >
strset_variable_acc
	[Variables& vars] 
	returns [boost::shared_ptr<VariableAccessor<StrSet> > strset_acc]
	:	STR_PREFIX n: SYMBOL {
		vars.get_put<StrSet>(str_token_rem_grav(n));
		
		VariableAccessor<StrSet> acc = 
			vars.create_accessor<StrSet>(str_token_rem_grav(n));

		strset_acc.reset(new VariableAccessor<StrSet>(acc));
	}
;

// Vargetter for StrSet variable. This rule wrapped strset_variable_acc.
// Returns boost::shared_ptr<VarGetter<StrSet> > 
strset_variable 
	[Variables& vars] 
	returns [boost::shared_ptr<VarGetter<StrSet> > op]
{
	boost::shared_ptr<VariableAccessor<StrSet> > strset_acc;
}
	: strset_acc = strset_variable_acc [vars] {
		op.reset(new VarGetter<StrSet>(*strset_acc));
	}
;

// ----------------------------------------------------------------------------
// Symbol set: $t:name
// Get symset variable (however, before put into) from variables
// Returns boost::shared_ptr<VariableAccessor<TSet> >
symset_variable_acc 
	[Variables& vars] 
	returns [boost::shared_ptr<VariableAccessor<TSet> > symset_acc]
	: TST_PREFIX n: SYMBOL { 
			vars.get_put<TSet>(str_token_rem_grav(n)); 	

			VariableAccessor<TSet> acc = 
				vars.create_accessor<TSet>(str_token_rem_grav(n));

			symset_acc.reset(new VariableAccessor<TSet>(acc));
	}
;

// Vargetter for symbol set variable. This rule wrapped symset_variable_acc
// Returns boost::shared_ptr<VarGetter<TSet> > 
symset_variable 
	[Variables& vars] 
	returns [boost::shared_ptr<VarGetter<TSet> > op]
{
	boost::shared_ptr<VariableAccessor<TSet> > symset_acc;
}
	: symset_acc = symset_variable_acc [vars] {
			op.reset(new VarGetter<TSet>(*symset_acc));
	}
;

// ----------------------------------------------------------------------------
// Bool: $b:name
// Get bool variable (however, before put into) from variables
// Returns boost::shared_ptr<VariableAccessor<Bool> >
bool_variable_acc
	[Variables& vars] 
	returns [boost::shared_ptr<VariableAccessor<Bool> > bool_acc]
	: BOOL_PREFIX n: SYMBOL { 
			vars.get_put<Bool>(str_token_rem_grav(n));

			VariableAccessor<Bool> acc = 
				vars.create_accessor<Bool>(str_token_rem_grav(n));

			bool_acc.reset(new VariableAccessor<Bool>(acc));
	}
;

// Vargetter for bool variable. It is only wrapper for bool_variable_acc
// Returns boost::shared_ptr<VarGetter<Bool> >
bool_variable
	[Variables& vars] 
	returns [boost::shared_ptr<VarGetter<Bool> > op]
{
	boost::shared_ptr<VariableAccessor<Bool> > bool_acc;
}
	: bool_acc = bool_variable_acc [vars] {
			op.reset(new VarGetter<Bool>(*bool_acc));
	}
;

// ----------------------------------------------------------------------------
// Match: $m:name
// Get mach vector variable from variavles (before put into variables)
// Returns boost::shared_ptr<VariableAccessor<Match> >
match_vector_variable_acc
	[Variables& vars]
	returns [boost::shared_ptr<VariableAccessor<Match> > mvv_acc]
	: MATCH_VECTOR_PREFIX n: SYMBOL {
			vars.get_put<Match>(str_token_rem_grav(n));

			VariableAccessor<Match> acc = 
				vars.create_accessor<Match>(str_token_rem_grav(n));

			mvv_acc.reset(new VariableAccessor<Match>(acc));
	}
;

// Vargetter for the match vector variavle. Wrapper for match_vector_variable_acc
// Returns boost::shared_ptr<VarGetter<Match> >
match_vector_variable
	[Variables& vars]
	returns [boost::shared_ptr<VarGetter<Match> > mvv]
{
	boost::shared_ptr<VariableAccessor<Match> > mvv_acc;
}
	: mvv_acc = match_vector_variable_acc [vars] {
		mvv.reset(new VarGetter<Match>(*mvv_acc));
	}
;

///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////
// OPERATORS
///////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////
// Symbol set (tagset) operators
// Returns boost::shared_ptr<Function<TSet> >
///////////////////////////////////////////////////////////////////////////////
symset_operator
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > ret]
	: ret = symset_condition [tagset, vars]
	| {LA(1)==SYMBOL && LA(2)==LBRACKET}? (ret = symset_getsymbol [tagset, vars])
	| ret = symset_var_val [tagset, vars]
	| ret = symset_class   [tagset, vars]
	| ret = symset_range   [tagset, vars]
	| ret = symset_catflt  [tagset, vars]
	| ret = symset_agrflt  [tagset, vars]
	//
	| LPAREN ret = symset_operator [tagset, vars] RPAREN
;

// ----------------------------------------------------------------------------
// It's wrapper for symset variable and symset value.
symset_var_val
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > op]
	: op = symset_variable [vars]
	| op = symset_value    [tagset]
;

// ----------------------------------------------------------------------------
// Condition of the symset value:
// 	if (Bool, TSet, TSet)
// 	? TSet ? Bool : {}
symset_condition
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > op]
{
	boost::shared_ptr<Function<Bool> > test;
	boost::shared_ptr<Function<TSet> > p_true, p_false;
}
	: "if" LPAREN test  = bool_operator [tagset, vars] COMMA 
							p_true  = symset_operator  [tagset, vars] 
							(COMMA p_false = symset_operator [tagset, vars])? 
		RPAREN {
			if (p_false) {
				op.reset(new Conditional<TSet>(test, p_true, p_false));
			}
			else {
				op.reset(new Conditional<TSet>(test, p_true));
			}
		}
	| Q_MARK 
			(p_true = symset_operator [tagset, vars])
		Q_MARK 
			(test = bool_operator [tagset, vars]) {
			op.reset(new Conditional<TSet>(test, p_true));
		}
;

// ----------------------------------------------------------------------------
// GetSymbol operator may be cas, m1, f, sg...
// WARNING! This rule can throw ParserException! Be careful!
symset_getsymbol
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > op]
{
	Corpus2::Tag tag;
	boost::shared_ptr<Wccl::Function<Position> > position;
}
	: t: SYMBOL LBRACKET position = position_operator [tagset, vars] RBRACKET {
			try {
				tag = tagset.parse_symbol(str_token_rem_grav(t));
			}
			catch(Corpus2::TagParseError &e) {
				throw(ParserException(e.info()));
			}
				
			op.reset(new Wccl::GetSymbols(tag, position));
		}
;

// ----------------------------------------------------------------------------
// Class operator.
symset_class 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > ret]
{
	boost::shared_ptr<Function<Position> > pos;
}
	: "class" LBRACKET pos = position_operator [tagset, vars] RBRACKET { 
		ret.reset(new GetWordClass(pos));
	}
;

// ----------------------------------------------------------------------------
// Range operator: range(class, begin, end) or range({...}, begin, end)
symset_range
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<TSet> > ret]
{
	boost::shared_ptr<TSet> tset;
	boost::shared_ptr<Function<Position> > p1, p2;
}
	: "range" LPAREN
			(tset = symset_literal [tagset] | tag_class: "class") COMMA
			p1  = position_operator [tagset, vars] COMMA 
			p2  = position_operator [tagset, vars] 
		RPAREN {
			if (tag_class) {
				ret.reset(new GetSymbolsInRange(Corpus2::Tag(-1), p1, p2));
			}
			else {
		 		ret.reset(new GetSymbolsInRange(tset->get_value(), p1, p2));
			}
		}
;

// ----------------------------------------------------------------------------
// Catflt operator
symset_catflt
  [const Corpus2::Tagset& tagset, Variables& vars]
  returns [boost::shared_ptr<Function<TSet> > ret]
{
  boost::shared_ptr<Function<TSet> > selector, mask;
  boost::shared_ptr<Function<Position> > position;
}
  : "catflt" LPAREN 
			position = position_operator [tagset, vars] COMMA
			selector = symset_operator   [tagset, vars] COMMA
			mask    = symset_operator    [tagset, vars] 
  	RPAREN {
  	  ret.reset(new CatFilter(position, selector, mask));
  	}
;

// ----------------------------------------------------------------------------
// Agrflt operator
symset_agrflt
  [const Corpus2::Tagset& tagset, Variables& vars]
  returns [boost::shared_ptr<Function<TSet> > ret]
{
  boost::shared_ptr<Function<TSet> > attr, mask;
  boost::shared_ptr<Function<Position> > lpos, rpos;
}
  : "agrflt" LPAREN 
			lpos = position_operator [tagset, vars] COMMA
			rpos = position_operator [tagset, vars] COMMA
			attr = symset_operator   [tagset, vars] COMMA
			mask = symset_operator   [tagset, vars] 
  	RPAREN {
  	  ret.reset(new AgrFilter(lpos, rpos, attr, mask, tagset));
  	}
;


///////////////////////////////////////////////////////////////////////////////
// Position operator
// Returns boost::shared_ptr<Function<Position> >
///////////////////////////////////////////////////////////////////////////////
position_operator
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<Position> > ret]
	: 
	( ret = position_var_val   [vars]
	| ret = position_condition [tagset, vars]
	| LPAREN ret = position_operator [tagset, vars] RPAREN
	) 
	( // if there is SIGNED_INT after the position, it is actually a relative position
		i: SIGNED_INT {
			ret.reset(new RelativePosition(ret, token_ref_to_int(i)));
		}
	)?
;

// ----------------------------------------------------------------------------
// Wrapper for position variable and position value
position_var_val
	[Variables& vars]
	returns [boost::shared_ptr<Function<Position> > ret]
	: ret = position_value
	| ret = position_variable [vars]
;

// ----------------------------------------------------------------------------
// Condition of the position value
// if (Bool, Position, Position)
// ? Position ? Bool : 0
position_condition
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<Position> > op]
{
	boost::shared_ptr<Function<Bool> > test;
	boost::shared_ptr<Function<Position> > p_true, p_false;
}
	: "if" LPAREN test  = bool_operator [tagset, vars] COMMA 
							p_true  = position_operator [tagset, vars] 
							(COMMA p_false = position_operator [tagset, vars])? 
	RPAREN {
		if (p_false) {
			op.reset(new Conditional<Position>(test, p_true, p_false));
		}
		else {
			op.reset(new Conditional<Position>(test, p_true));
		}
	}
	| Q_MARK 
			p_true = position_operator [tagset, vars]
		Q_MARK 
			test = bool_operator [tagset, vars] {
			op.reset(new Conditional<Position>(test, p_true));
		}
;

///////////////////////////////////////////////////////////////////////////////
// Stiring operator
// Returns boost::shared_ptr<Function<StrSet> >
///////////////////////////////////////////////////////////////////////////////
strset_operator [const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
	: ret = strset_orth      [tagset, vars] 
	| ret = strset_base      [tagset, vars]
	| ret = strset_lower     [tagset, vars] 
	| ret = strset_upper     [tagset, vars]
	| ret = strset_affix     [tagset, vars] 
	| ret = strset_var_val   [tagset, vars] 
	| ret = strset_condition [tagset, vars]
	//
	| LPAREN ret = strset_operator [tagset, vars] RPAREN
;

// ----------------------------------------------------------------------------
// Orth operator.
strset_orth 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
{
	boost::shared_ptr<Function<Position> > pos;
}
	: "orth" LBRACKET pos = position_operator [tagset, vars] RBRACKET { 
			ret.reset(new GetOrth(pos));
	}
;

// ----------------------------------------------------------------------------
// Base operator.
strset_base 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
{
	boost::shared_ptr<Function<Position> > pos;
}
	: "base" LBRACKET pos = position_operator [tagset, vars] RBRACKET { 
		ret.reset(new GetLemmas(pos));
	}
;

// ----------------------------------------------------------------------------
// Lower operator.
strset_lower 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
{
	boost::shared_ptr<Function<StrSet> > o_ret;
}
	: "lower" LPAREN o_ret = strset_operator [tagset, vars] RPAREN {
		ret.reset(new ToLower(o_ret));
	}
;

// ----------------------------------------------------------------------------
// Upper operator.
strset_upper 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
{
	boost::shared_ptr<Function<StrSet> > o_ret;
}
	: "upper" LPAREN o_ret = strset_operator [tagset, vars] RPAREN {
		ret.reset(new ToUpper(o_ret));
	}
;

// ----------------------------------------------------------------------------
// Affix operator.
strset_affix 
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > ret]
{
	int offset = 0;
	boost::shared_ptr<Function<StrSet> > o_ret;
}
	: "affix" LPAREN 
			o_ret = strset_operator [tagset, vars] COMMA offset = number 
		RPAREN {
			ret.reset(new Affix(o_ret, offset));
		}
;

// ----------------------------------------------------------------------------
// Wrapper ofr strset value and strset variable
strset_var_val
	[const Corpus2::Tagset& /*tagset*/, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > op]
	: op = strset_value 
	| op = strset_variable [vars]
;

// ----------------------------------------------------------------------------
// Condition of the strset value
// if (Bool, StrSet, StrSet)
// ? StrSet ? Bool : []
strset_condition
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<StrSet> > op]
{
	boost::shared_ptr<Function<Bool> > test;
	boost::shared_ptr<Function<StrSet> > p_true, p_false;
}
	: "if" LPAREN test  = bool_operator [tagset, vars] COMMA 
							p_true  = strset_operator   [tagset, vars] 
							(COMMA p_false = strset_operator [tagset, vars])? 
	RPAREN {
		if (p_false) {
			op.reset(new Conditional<StrSet>(test, p_true, p_false));
		}
		else {
			op.reset(new Conditional<StrSet>(test, p_true));
		}
	}
	| Q_MARK 
			p_true = strset_operator [tagset, vars]
		Q_MARK 
			test = bool_operator [tagset, vars] {
			op.reset(new Conditional<StrSet>(test, p_true));
		}
;

///////////////////////////////////////////////////////////////////////////////
// Boool operator 
// Returns boost::shared_ptr<Function<Bool> >
///////////////////////////////////////////////////////////////////////////////
bool_operator
	[const Corpus2::Tagset& tagset, Variables& vars]
	returns [boost::shared_ptr<Function<Bool> > ret]
	: ret = bool_and        [tagset, vars]  
	| ret = bool_or         [tagset, vars]  
	| ret = bool_nor        [tagset, vars]  
	| ret = bool_var_val    [tagset, vars]