検索式を構文解析するPythonモジュール
細かいチューニングがしたかったのでpyparsingを使って自分で書いた。
一応仕事で使うためのもの（でも間違いなくオーバースペック)。
# -*- coding: utf-8 -*- # # Copyright (c) 2007 Nozomu Kaneko # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # """Yet another search query parser This search query parser uses the excellent Pyparsing module (http://pyparsing.sourceforge.net/) to parse search queries by users. You can provide an acceptor to the parser in order to construct parse results in the specific form. For example, you can construct SQLAlchemy `ClauseElement` objects in your acceptor so that search queries can be translated directly into SQL expressions. Lots of parts of this script are based on the following scripts: * aql.py http://pyparsing-public.wikispaces.com/space/showimage/aql.py * searchparser.py http://pyparsing.wikispaces.com/space/showimage/searchparser.py Thanks to the authors of the scripts and the pyparsing module. """ from pyparsing import * def parser(): _eq_ = Suppress("=") _ne_ = Suppress("!=") _lt_ = Suppress("<") _gt_ = Suppress(">") _le_ = Suppress("<=") _ge_ = Suppress(">=") _lparen_ = Suppress("(") _rparen_ = Suppress(")") _sign_ = Word("+-", exact=1) Int = Combine(Optional(_sign_) + Word(nums)) \ .setParseAction(lambda l, s, t: int(t[0])) \ .setResultsName('int') Float = Combine(Optional(_sign_) + ( Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums)) ) + Optional("E" + Optional(_sign_) + Word(nums))) \ .setParseAction(lambda l, s, t: float(t[0])) \ .setResultsName('float') Number = Float | Int String = quotedString.copy() \ .setParseAction(removeQuotes) \ .setResultsName('string') Constant = Float | Int | String _not_ = Keyword("not", caseless=True).suppress() _and_ = Keyword("and", caseless=True).suppress() _or_ = Keyword("or", caseless=True).suppress() _dotdot_ = Keyword("..").suppress() _in_ = Keyword("in", caseless=True).suppress() Lval = Word(alphanums + "_.") Rval = Float | Int | String | Lval RvalList = ( _lparen_ + Group(delimitedList(Rval)).setResultsName('vals') + _rparen_ ) Clause = Forward() ComplexClause = Forward() Range = ( _lparen_ + Optional(Number, default=None).setResultsName('min') + _dotdot_ + Optional(Number, default=None).setResultsName('max') + _rparen_ ) Clause << ( Group( Lval + _eq_ + Rval ).setResultsName('eq') | Group( Lval + _ne_ + Rval ).setResultsName('ne') | Group( Lval + _lt_ + Rval ).setResultsName('lt') | Group( Lval + _gt_ + Rval ).setResultsName('gt') | Group( Lval + _le_ + Rval ).setResultsName('le') | Group( Lval + _ge_ + Rval ).setResultsName('ge') | Group( Lval + _eq_ + Range ).setResultsName('range') | Group( Lval + _in_ + RvalList ).setResultsName('in') | Group( _lparen_ + ComplexClause + _rparen_).setResultsName('paren') ) ComplexClause << \ ( Group( _not_ + Clause ).setResultsName('not') | Group( Clause + _or_ + ComplexClause ).setResultsName('or') | Group( Clause + _and_ + ComplexClause ).setResultsName('and') | Clause ) BareWord = CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS + '"()') Term = Group( String | BareWord ).setResultsName('term') Prefix = ( Regex(r"([A-Za-z][A-Za-z0-9_\-\.]*):") \ .setParseAction(lambda l, s, t: t[0][:-1]) | _sign_ ) Directive = Group( Prefix + ( String | BareWord )).setResultsName('directive') QueryElement = Forward() QueryElementList = Forward() QueryElement << ( Directive | ComplexClause | ( _lparen_ + QueryElementList + _rparen_ ) | Term ) QueryElementList << \ ( Group( QueryElement + _or_ + QueryElementList).setResultsName('or') | Group( QueryElement + Optional(_and_) + QueryElementList).setResultsName('and') | QueryElement ) Query = QueryElementList + stringEnd return Query class SearchQueryParser(object): """ >>> p = SearchQueryParser(NullAcceptor()) >>> p("a") 'a' >>> p(u"\u30c6\u30b9\u30c8 test *") ('and', u'\u30c6\u30b9\u30c8', u'test', u'*') >>> p('x y z') ('and', 'x', 'y', 'z') >>> p('x and y or z') ('and', 'x', ('or', 'y', 'z')) >>> p('(a or b)') ('or', 'a', 'b') >>> p('(a or b) and (c or d)') ('and', ('or', 'a', 'b'), ('or', 'c', 'd')) >>> p('x +y -z') ('and', 'x', ('directive', '+', 'y'), ('directive', '-', 'z')) >>> p('"x y z"') 'x y z' >>> p(u'\u3042 \u3044 \u3046') ('and', u'\u3042', u'\u3044', u'\u3046') >>> p(u'\u3042 +\u3044 -\u3046') ('and', u'\u3042', ('directive', u'+', u'\u3044'), ('directive', u'-', u'\u3046')) >>> p('x = 1.0') ('=', 'x', 1.0) >>> p('x < 1') ('<', 'x', 1) >>> p('x = "1"') ('=', 'x', '1') >>> p('x = ""') ('=', 'x', '') >>> p('x = 1 and y != 2') ('and', ('=', 'x', 1), ('!=', 'y', 2)) >>> p('z = "foo" and not x = 3') ('and', ('=', 'z', 'foo'), ('not', ('=', 'x', 3))) >>> p('z >= 3 or z <= 5') ('or', ('>=', 'z', 3), ('<=', 'z', 5)) >>> p('z <= 3 or z >= 4') ('or', ('<=', 'z', 3), ('>=', 'z', 4)) >>> p('x = ( .. )') ('range', 'x', None, None) >>> p('x = ( 1 .. 3.0 )') ('range', 'x', 1, 3.0) >>> p('x = ( 1.0 .. )') ('range', 'x', 1.0, None) >>> p('x in ( 1, 2, 3 ) or z >= 4') ('or', ('in', 'x', [1, 2, 3]), ('>=', 'z', 4)) >>> p('z <= 3 and z = ( 1 .. 2 )') ('and', ('<=', 'z', 3), ('range', 'z', 1, 2)) >>> p('z = "foo" or (z != 3.0 and fiz != "fozz")') ('or', ('=', 'z', 'foo'), ('and', ('!=', 'z', 3.0), ('!=', 'fiz', 'fozz'))) >>> p("foo:val test x=y") ('and', ('directive', 'foo', 'val'), 'test', ('=', 'x', 'y')) >>> p('test foo:"a b c" x=y') ('and', 'test', ('directive', 'foo', 'a b c'), ('=', 'x', 'y')) """ def __init__(self, acceptor): self.parser = parser() self.acceptor = acceptor def __call__(self, query): r = self.parser.parseString(query) return self.visit(r[0]) def visit(self, arg): methods = { 'and': self.acceptor.accept_and, 'or': self.acceptor.accept_or, 'not': self.acceptor.accept_not, 'eq': self.acceptor.accept_eq, 'ne': self.acceptor.accept_ne, 'lt': self.acceptor.accept_lt, 'gt': self.acceptor.accept_gt, 'le': self.acceptor.accept_le, 'ge': self.acceptor.accept_ge, 'range': self.acceptor.accept_range, 'in': self.acceptor.accept_in, 'vals': lambda *kw: list(kw), 'paren': self.visit, } if not isinstance(arg, ParseResults): return arg name = arg.getName() if name == 'directive': return self.acceptor.accept_directive(*arg) elif name == 'term': return self.acceptor.accept_term(*arg) else: f = methods[name] return f(*map(self.visit, arg)) class NullAcceptor(object): """ Example acceptor for SearchQueryParser. It is also used in doctest of SearchQueryParser. You don't have to override this class, but you should define your own acceptor with all the `accept_*` methods. """ def accept_and(self, lhs, rhs): if isinstance(rhs, tuple) and rhs[0] == 'and': return ('and', lhs) + rhs[1:] else: return ('and', lhs, rhs) def accept_or(self, lhs, rhs): if isinstance(rhs, tuple) and rhs[0] == 'or': return ('or', lhs) + rhs[1:] else: return ('or', lhs, rhs) def accept_not(self, clause): return ('not', clause) def accept_eq(self, lhs, rhs): return ('=', lhs, rhs) def accept_ne(self, lhs, rhs): return ('!=', lhs, rhs) def accept_lt(self, lhs, rhs): return ('<', lhs, rhs) def accept_gt(self, lhs, rhs): return ('>', lhs, rhs) def accept_le(self, lhs, rhs): return ('<=', lhs, rhs) def accept_ge(self, lhs, rhs): return ('>=', lhs, rhs) def accept_range(self, lval, min, max): return ('range', lval, min, max) def accept_in(self, lval, rval): return ('in', lval, rval) def accept_directive(self, lval, rval): return ('directive', lval, rval) def accept_term(self, term): return term def repl(): import cmd class Repl(cmd.Cmd): def __init__(self, **kw): cmd.Cmd.__init__(self, **kw) self.parser = SearchQueryParser(NullAcceptor()) def onecmd(self, line): if line == '': return True print self.parser(line) return False Repl().cmdloop() if __name__ == '__main__': import sys if len(sys.argv) == 1: repl() elif len(sys.argv) > 1 and sys.argv[1] == 'test': import doctest doctest.testmod() else: warn("Invalid arguments") __all__ = [ 'SearchQueryParser', 'NullAcceptor' ]
ソースコード中のコメントにも書いてあるように、検索式から直接SQL文を生成することが最終目標。次は構文解析の結果からSQLAlchemyのClauseElementを生成する部分を書く。
