Inserted up to date Gadfly release, along with merged hack to make it

work properly under Zope... this should show up in the normal release
of Gadfly as soon as ARW releases another one :-)

lib/python/Products/ZGadflyDA/gadfly/COPYRIGHT

+
+The following copyright is modified from the python copyright.
+
+Copyright Notice
+----------------
+
+The kjParsing source is copyrighted, but you can freely use and copy it
+as long as you don't change or remove the copyright:
+
+Copyright Aaron Robert Watters, 1994
+
+                        All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its 
+documentation for any purpose and without fee is hereby granted, 
+provided that the above copyright notice appears in all copies and that
+both that copyright notice and this permission notice appear in 
+supporting documentation.
+
+AARON ROBERT WATTERS DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL AARON ROBERT WATTERS BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+Signature
+---------
+        Aaron Robert Watters
+        Department of Computer and Information Sciences
+        New Jersey Institute of Technology
+        University Heights
+        Newark, NJ 07102
+                phone (201)596-2666
+		fax (201)596-5777
+		home phone (908)545-3367
+		email: aaron@vienna.njit.edu

lib/python/Products/ZGadflyDA/gadfly/DLispShort.py

+# Grammar generation 
+# for lisp lists with strings, ints, vars, print, and setq
+
+# set this variable to regenerate the grammar on each load
+REGENERATEONLOAD = 1
+
+import string
+GRAMMARSTRING ="""
+       Value ::  ## indicates Value is the root nonterminal for the grammar
+         @R SetqRule :: Value >> ( setq var Value )
+         @R ListRule :: Value >> ( ListTail
+         @R TailFull :: ListTail >> Value ListTail
+         @R TailEmpty :: ListTail >> )
+         @R Varrule :: Value >> var
+         @R Intrule :: Value >> int
+         @R Strrule :: Value >> str
+         @R PrintRule :: Value >> ( print Value )
+"""
+COMPILEDFILENAME = "TESTLispG.py"
+MARSHALLEDFILENAME = "TESTLispG.mar"
+LISPCOMMENTREGEX = ";.*"
+INTREGEX = "["+string.digits+"]+"
+STRREGEX = '"[^\n"]*"'
+VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
+
+### declare interpretation functions and regex's for terminals
+def intInterp( str ):
+    return string.atoi(str)
+def stripQuotes( str ):
+    return str[1:len(str)-1]
+def echo(string):
+    return string
+def DeclareTerminals(Grammar):
+    Grammar.Addterm("int", INTREGEX, intInterp)
+    Grammar.Addterm("str", STRREGEX, stripQuotes)
+    Grammar.Addterm("var", VARREGEX, echo)
+
+### declare the rule reduction interpretation functions.
+def EchoValue( list, Context ):
+    return list[0]
+def VarValue( list, Context ):
+    varName = list[0]
+    if Context.has_key(varName):
+       return Context[varName]
+    else:
+       raise NameError, "no such lisp variable in context "+varName
+def NilTail( list, Context ):
+    return []
+def AddToList( list, Context ):
+    return [ list[0] ] + list[1]
+def MakeList( list, Context ):
+    return list[1]
+def DoSetq( list, Context):
+    Context[ list[2] ] = list[3]
+    return list[3]
+def DoPrint( list, Context ):
+    print list[2]
+    return list[2]
+def BindRules(Grammar):
+    Grammar.Bind( "Intrule", EchoValue )
+    Grammar.Bind( "Strrule", EchoValue )
+    Grammar.Bind( "Varrule", VarValue )
+    Grammar.Bind( "TailEmpty", NilTail )
+    Grammar.Bind( "TailFull", AddToList )
+    Grammar.Bind( "ListRule", MakeList )
+    Grammar.Bind( "SetqRule", DoSetq )
+    Grammar.Bind( "PrintRule", DoPrint )
+
+# This function generates the grammar and dumps it to a file.  
+def GrammarBuild():
+    import kjParseBuild
+    LispG = kjParseBuild.NullCGrammar()
+    LispG.SetCaseSensitivity(0) # grammar is not case sensitive for keywords
+    DeclareTerminals(LispG)
+    LispG.Keywords("setq print")
+    LispG.punct("().")
+    LispG.Nonterms("Value ListTail")
+    LispG.comments([LISPCOMMENTREGEX])
+    LispG.Declarerules(GRAMMARSTRING)
+    LispG.Compile()
+
+    print "dumping as python to "+COMPILEDFILENAME
+    outfile = open(COMPILEDFILENAME, "w")
+    LispG.Reconstruct("LispG",outfile,"GRAMMAR")
+    outfile.close()
+
+    print "dumping as binary to "+MARSHALLEDFILENAME
+    outfile = open(MARSHALLEDFILENAME, "w")
+    LispG.MarshalDump(outfile)
+    outfile.close()
+
+    BindRules(LispG)
+    return LispG
+
+# this function initializes the compiled grammar from the generated file.  
+def LoadLispG():
+    import TESTLispG
+    # reload to make sure we get the most recent version!
+    # (only needed when debugging the grammar).
+    reload(TESTLispG)
+    LispG = TESTLispG.GRAMMAR()
+    DeclareTerminals(LispG)
+    BindRules(LispG)
+    return LispG
+
+def unMarshalLispG():
+    import kjParser
+    infile = open(MARSHALLEDFILENAME, "r")
+    LispG = kjParser.UnMarshalGram(infile)
+    infile.close()
+    DeclareTerminals(LispG)
+    BindRules(LispG)
+    return LispG
+
+########## test the grammar generation
+if REGENERATEONLOAD:
+   print "(re)generating the LispG grammar in file TESTLispG.py"
+   Dummy = GrammarBuild()
+   print "(re)generation done."
+
+print "loading grammar as python"
+LispG = LoadLispG()
+### declare an initial context, and do some tests.
+Context = { 'x':3 }
+test1 = LispG.DoParse1( '()', Context)
+test2 = LispG.DoParse1( '(123)', Context)
+test3 = LispG.DoParse1( '(x)', Context)
+test4 = LispG.DoParse1( '" a string "', Context)
+test5 = LispG.DoParse1( '(setq y (1 2 3) )', Context )
+test6 = LispG.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
+test7str = """
+  ; this is a lisp comment
+  (setq abc (("a" x)
+             ("b" (setq d 12))
+             ("c" y) ) ; another lisp comment
+  )
+"""
+test7 = LispG.DoParse1( test7str, Context)
+test8 = LispG.DoParse1( '(print (1 x d))', Context)
+
+print "unmarshalling the grammar"
+LispG2 = unMarshalLispG()
+### declare an initial context, and do some tests.
+Context = { 'x':3 }
+test1 = LispG2.DoParse1( '()', Context)
+test2 = LispG2.DoParse1( '(123)', Context)
+test3 = LispG2.DoParse1( '(x)', Context)
+test4 = LispG2.DoParse1( '" a string "', Context)
+test5 = LispG2.DoParse1( '(setq y (1 2 3) )', Context )
+test6 = LispG2.DoParse1( '(SeTq x ("a string" "another" 0))', Context )
+test7str = """
+  ; this is a lisp comment
+  (setq abc (("a" x)
+             ("b" (setq d 12))
+             ("c" y) ) ; another lisp comment
+  )
+"""
+test7 = LispG2.DoParse1( test7str, Context)
+test8 = LispG2.DoParse1( '(print (1 x d))', Context)
+

lib/python/Products/ZGadflyDA/gadfly/DumbLispGen.py

+
+#
+# test for kjParseBuild module automatic parser generation
+#
+# lisp lists with strings, ints, vars, and setq
+
+import string
+
+### The string representation for the grammar.
+### Since this is used only by GrammarBuild()
+### it could be put in a separate file with GrammarBuild()
+### to save space/load time after Grammar compilation.
+###
+GRAMMARSTRING ="""
+       Value ::  ## indicates Value is the root nonterminal for the grammar
+         @R SetqRule :: Value >> ( setq var Value )
+         @R ListRule :: Value >> ( ListTail
+         @R TailFull :: ListTail >> Value ListTail
+         @R TailEmpty :: ListTail >> )
+         @R Varrule :: Value >> var
+         @R Intrule :: Value >> int
+         @R Strrule :: Value >> str
+"""
+
+### the name of the file in which to create the compiled
+### grammar declarations
+COMPILEDFILENAME = "TESTLispG2.py"
+
+### declare comment form(s) as regular expressions 
+LISPCOMMENTREGEX = ";.*"
+
+### declare regular expression string constants for terminals
+
+#integer terminal::::::: 
+INTREGEX = "["+string.digits+"]+"
+
+#string terminal:::::::: 
+STRREGEX = '"[^\n"]*"'
+
+#var terminal:::::::: 
+VARREGEX = "["+string.letters+"]["+string.letters+string.digits+"]*"
+
+### declare interpretation functions for terminals
+
+# int interpretation function: translates string to int: 
+# Could use string.atoi without the extra level of indirection 
+# but for demo purposes here it is.  
+# 
+def intInterp( str ):
+   return string.atoi(str)
+
+# interpretation function for strings strips off the surrounding quotes.  
+def stripQuotes( str ):
+   if len(str)<2:
+      TypeError, "string too short?"
+   return str[1:len(str)-1]
+
+# interpretation function for vars just returns the recognized string
+def echo(string):
+   return string
+
+# This function declares the nonterminals both in the 
+# "grammar generation phase" and in loading the compiled 
+# grammar after generation 
+# 
+def DeclareTerminals(Grammar):
+   Grammar.Addterm("int", INTREGEX, intInterp)
+   Grammar.Addterm("str", STRREGEX, stripQuotes)
+   Grammar.Addterm("var", VARREGEX, echo)
+
+### declare the rule reduction interpretation functions.
+
+# EchoValue() serves for Intrule and Strrule, since 
+# we just want to echo the value returned by the 
+# respective terminal interpretation functions.  
+# 
+# Parser delivers list of form [ interpreted_value ] 
+def EchoValue( list, Context ):
+   if len(list)!=1:
+      raise TypeError, "this shouldn't happen! (1)"
+   return list[0]
+
+# for Varrule interpreter must try to look up the value 
+# in the Context dictionary 
+# 
+# Parser delivers list of form [ var_name ] 
+def VarValue( list, Context ):
+   if len(list)!=1:
+      raise TypeError, "Huh? (2)"
+   varName = list[0]
+   if Context.has_key(varName):
+      return Context[varName]
+   else:
+      raise NameError, "no such lisp variable in context "+varName
+
+# for an empty tail, return the empty list 
+# 
+# Parser delivers list of form [")"] 
+def NilTail( list, Context ):
+   if len(list) != 1 or list[0] != ")":
+      return TypeError, "Bad reduction?"
+   return []
+
+# For a full tail, add the new element to the front of the list 
+# 
+# Parser delivers list of form [Value, TailValue] 
+def AddToList( list, Context ):
+   if len(list) !=2:
+      return TypeError, "Bad reduction?"
+   return [ list[0] ] + list[1]
+
+# For a list, simply return the list determined by the tail 
+# 
+# Parser delivers list of form ["(", TailValue ] 
+def MakeList( list, Context ):
+   if len(list)!=2 or list[0]!="(":
+      raise TypeError, "Bad reduction? (3)"
+   return list[1]
+
+# For a setq, declare a new variable in the Context dictionary 
+# 
+# Parser delivers list of form # ["(", "setq", varName, Value, ")"] 
+def DoSetq( list, Context):
+   if len(list) != 5\
+     or list[0] != "("\
+     or list[1] != "setq"\
+     or list[4] != ")":
+      print list
+      raise TypeError, "Bad reduction? (4)"
+   VarName = list[2]
+   if type(VarName) != type(''):
+      raise TypeError, "Bad var name? (5)"
+   Value = list[3]
+   # add or set the variable in the Context dictionary
+   Context[ VarName ] = Value
+   return Value
+
+# This function Binds the named rules of the Grammar string to their 
+# interpretation functions in a Grammar.  
+# 
+def BindRules(Grammar):
+    Grammar.Bind( "Intrule", EchoValue )
+    Grammar.Bind( "Strrule", EchoValue )
+    Grammar.Bind( "Varrule", VarValue )
+    Grammar.Bind( "TailEmpty", NilTail )
+    Grammar.Bind( "TailFull", AddToList )
+    Grammar.Bind( "ListRule", MakeList )
+    Grammar.Bind( "SetqRule", DoSetq )
+
+# This function generates the grammar and dumps it to a file.  
+# Since it will be used only once (after debugging), 
+# it probably should be put in another file save memory/load-time.  
+#
+# the result returned is a Grammar Object that can be used 
+# for testing/debugging purposes.  
+#
+# (maybe this should be made into a generic function?)  
+def GrammarBuild():
+    import kjParseBuild
+
+    # initialize a Null compilable grammar to define
+    LispG = kjParseBuild.NullCGrammar()
+    
+    # declare terminals for the grammar
+    DeclareTerminals(LispG)
+
+    # declare the keywords for the grammar
+    # defun is not used, included here for demo purposes only
+    LispG.Keywords("setq defun")
+
+    # Declare punctuations
+    # dot is not used here
+    LispG.punct("().")
+
+    # Declare Nonterms
+    LispG.Nonterms("Value ListTail")
+
+    # Declare comment forms
+    LispG.comments([LISPCOMMENTREGEX])
+
+    # Declare rules
+    LispG.Declarerules(GRAMMARSTRING)
+
+    # Compile the grammar
+    LispG.Compile()
+
+    # Write the grammar to a file except for
+    # the function bindings (which must be rebound)
+    outfile = open(COMPILEDFILENAME, "w")
+    LispG.Reconstruct("LispG",outfile,"GRAMMAR")
+    outfile.close()
+
+    # for debugging purposes only, bind the rules
+    # in the generated grammar
+    BindRules(LispG)
+
+    # return the generated Grammar
+    return LispG
+
+# this function initializes the compiled grammar from 
+# generated file.  
+def LoadLispG():
+    import TESTLispG2
+    # make sure we have most recent version (during debugging)
+    reload(TESTLispG2)
+    # evaluate the grammar function from generated file
+    LispG = TESTLispG2.GRAMMAR()
+    # bind the semantics functions
+    DeclareTerminals(LispG)
+    BindRules(LispG)
+    return LispG
+
+########## test grammar generation
+
+# do generation
+Dummy = GrammarBuild()
+
+# load the grammar from the file as LispG
+LispG = LoadLispG()
+
+# declare an initial context, and do some tests.
+Context = { "x":3 }
+test1 = LispG.DoParse1( "()", Context)
+test2 = LispG.DoParse1( "(123)", Context)
+test3 = LispG.DoParse1( "(x)", Context)
+test4 = LispG.DoParse1( '" a string "', Context)
+test5 = LispG.DoParse1( "(setq y (1 2 3) )", Context )
+test6 = LispG.DoParse1( '(setq x ("a string" "another" 0))', Context )
+test7str = """
+  ; this is a lisp comment
+  (setq abc (("a" x)
+             ("b" (setq d 12))
+             ("c" y) ) ; another lisp comment
+  )
+"""
+test7 = LispG.DoParse1( test7str, Context)

lib/python/Products/ZGadflyDA/gadfly/SQLTESTG.py

+# this was used for debugging null productions (a nearly full sql grammar
+# is available on request).
+
+#set this to automatically rebuild the grammar.
+REBUILD = 1
+MARSHALFILE = "SQLTEST.mar"
+
+SELECTRULES = """
+  ## highest level for select statement (not select for update)
+  select-statement ::
+  @R selectR :: select-statement >> 
+                   SELECT 
+                   from-clause
+                   where-clause
+                   group-by-clause
+                   having-clause
+  ## generalized to allow null from clause eg: select 2+2
+  @R fromNull :: from-clause >>
+  @R fromFull :: from-clause >> FROM 
+  @R whereNull :: where-clause >>
+  @R whereFull :: where-clause >> WHERE 
+  @R groupNull :: group-by-clause >>
+  @R groupFull :: group-by-clause >> GROUP BY 
+  @R havingNull :: having-clause >> 
+  @R havingFull :: having-clause >> HAVING
+  @R unionNull :: union-clause >> 
+  @R unionFull :: union-clause >> UNION 
+"""
+
+SELECTNONTERMS = """
+  select-statement
+  all-distinct select-list table-reference-list
+  where-clause group-by-clause having-clause union-clause
+  maybe-order-by
+  search-condition column-list maybe-all order-by-clause
+  column-name from-clause
+"""
+# of these the following need resolution
+#   (select-list) (table-reference-list) 
+#   (search-condition) order-by-clause (column-name) 
+
+SELECTKEYWORDS = """
+  SELECT FROM WHERE GROUP BY HAVING UNION DISTINCT ALL AS 
+"""
+
+# test generation of the grammar
+def BuildSQLG():
+   import kjParseBuild
+   SQLG = kjParseBuild.NullCGrammar()
+   SQLG.SetCaseSensitivity(0)
+   SQLG.Keywords(SELECTKEYWORDS)
+   SQLG.Nonterms(SELECTNONTERMS)
+   # no comments yet
+   SQLG.Declarerules(SELECTRULES)
+   print "building"
+   SQLG.Compile()
+   print "marshaling"
+   outfile = open( MARSHALFILE, "w")
+   SQLG.MarshalDump(outfile)
+   outfile.close()
+   return SQLG
+
+# load function
+def LoadSQLG():
+   import kjParser
+   print "unmarshalling"
+   infile = open(MARSHALFILE, "r")
+   SQLG = kjParser.UnMarshalGram(infile)
+   infile.close()
+   return SQLG
+
+#### for testing
+if REBUILD:
+   SQLG0 = BuildSQLG()
+   print " rebuilt SQLG0 as compilable grammar"
+
+SQLG = LoadSQLG()
+print " build SQLG as reloaded grammar"
\ No newline at end of file

lib/python/Products/ZGadflyDA/gadfly/__init__.py

+# Package handling monstrocities
+
+import os
+import sqlwhere
+
+# Stuff filename into the namespace so that you don't have to hard code
+# this information ahead of time.  Should be portable across platforms.
+sqlwhere.filename = filename = __path__[0] + os.sep + 'sql.mar'
+
+# Yank the previous namespace in so that nothing breaks
+from gadfly import *

lib/python/Products/ZGadflyDA/gadfly/gfSQL.html

+<html>
+<head>
+<title>
+Gadfly SQL constructs
+</title>
+</head>
+<body bgcolor="#ffffff">
+
+<table>
+<tr>
+<td>
+<img src="gadfly.JPG">
+</td>
+<td>
+<h1>Gadfly SQL constructs</h1>
+</td></tr></table>
+
+<blockquote>
+This document describes SQL constructs supported by Gadfly.
+The presentation
+does not define the complete syntax -- see sqlgram.py for
+the precise syntax as BNF -- nor the complete semantics --
+see a good book on SQL for more detailed coverage of semantic
+(or use the source, Luke ;c) ).
+Also, please have a look at my
+<a href="http://mulder.rutgers.edu/~aaron/dbnotes.cgi">evolving
+database course notes</a> for more coverage of SQL.
+Examples of all supported constructs are also shown in the
+test suite source file gftest.py.
+This document is only
+a very brief guide, primarily of use to those who already
+understand something about SQL -- it is neither a tutorial
+nor a detailed discussion of syntax and semantics.
+</blockquote>
+
+<h1>The Standard, with omissions</h1>
+
+<p>
+Gadfly supports a large subset of ODBC 2.0 SQL.  The reason
+ODBC 2.0 is chosen is because it provides a fairly strong set of
+constructs, but does not include some of the more obscure
+features of other SQL standards which would be extremely
+difficult and complex to implement correctly
+(and perhaps, not used very frequently (?)).  
+<p>
+Supported features include views, groupings, aggregates,
+subquery expressions, quantified subquery comparisons,
+EXISTS, IN, UNION, EXCEPT, INTERSECT, searched mutations and
+Indices, among others (see below).
+
+<p>
+Some important omissions from ODBC 2.0 at this point are
+<pre>
+Nulls.
+Outer joins.
+CHECK conditions.
+Enforced data type constraints.
+Alter table (can't implement until NULLs arrive).
+Date, Time, and Interval data types
+</pre>
+It is hoped these will be implemented at some future time.
+<p>
+Less important omissions include
+<pre>
+Cursor based updates and deletes
+  (justification: if you really need them the db design
+   is flawed, and it's possible to use python instead).
+LIKE string predicate
+  (justification: use Python regexes in python code).
+Users and permissions
+  (justification: Gadfly is not intended for full multiuser
+   use at this time).
+</pre>
+These may or may not be implemented at some future time.
+
+<h1>Statements</h1>
+
+All interaction with SQL databases is mediated by
+SQL statements, or statement sequences.  Statement
+sequences are statements separated by semicolons.
+SQL keywords and user defined names are not case
+sensitive (but string values are, of course).
+<p>
+SQL statements include the following.
+
+<h3>Select Statement</h3>
+The select statement derives a table from tables
+in the database.  It's general form is
+<pre>
+sub_query
+     optorder_by
+</pre>
+Where sub_query is given by
+<pre>
+SELECT alldistinct select_list
+     FROM table_reference_list
+     optwhere 
+     optgroup 
+     opthaving 
+     optunion
+</pre>
+Read the statement:
+<pre>
+SELECT [DISTINCT|ALL] expressions or *
+FROM tables
+[WHERE condition]
+[GROUP BY group-expressions]
+[HAVING aggregate-condition]
+[union-clause]
+[ORDER BY columns]
+</pre>
+as follows: 
+<pre>
+1) Make all combinations of rows from the tables (FROM line)
+2) Eliminate those combinations not satisfying condition (WHERE line)
+3) (if GROUP present) form aggregate groups that match on group-expressions
+4) (if HAVING present) eliminate aggregate groups that don't satisfy
+   the aggregate-condition.
+5) compute the columns to keep (SELECT line).
+6) (if union-clause present) combine (union, except, intersect)
+   the result with the result of another select statement.
+7) if DISTINCT, throw out redundant entries.
+8) (if ORDER present) order the result by the columns (ascending
+   or descending as specified, with precedence as listed).
+</pre>
+This reading has little to do with the actual implementation,
+but the answer produced should match this intuitive reading.
+
+<h3>Create and drop table</h3>
+
+The create and drop table constructs
+initialize and destroy a table structure, respectively.
+<pre>
+CREATE TABLE user_defined_name ( colelts )
+
+DROP TABLE user_defined_name
+</pre>
+The colelts declare the names of the columns for
+the table and their data types.  The data types are
+not checked or enforced in any way at this time.
+
+<h3>Table mutations (INSERT, UPDATE, DELETE)</h3>
+
+Insert, Update, and Delete statements insert rows
+into tables, modify rows in tables in place, or
+remove rows from tables respectively.
+<pre>
+INSERT INTO table_name optcolids insert_spec
+
+DELETE FROM user_defined_name optwhere
+
+UPDATE user_defined_name
+     SET assns
+     optwhere
+</pre>
+The insert statement has two variants (in this implementation)
+INSERT sub-select and INSERT VALUES.  
+<pre>
+insert into r (a,b,c) select a,b,c from s
+
+insert into r (a,b,c) values (1,2,3)
+</pre>
+The first inserts
+the result of a SELECT statement into the target table
+and the other inserts explicit values (which may be dynamic
+parameters, see below).
+<P>
+Cursor based updates are not supported at the SQL level,
+eg
+<pre>
+  update r set a=1 where current of curs
+</pre>
+is not supported.
+
+<h3>Indices</h3>
+The create and drop index statements initialize and
+destroy index structures respectively.
+<pre>
+CREATE INDEX user_defined_name
+     ON user_defined_name
+     ( namelist )
+
+DROP INDEX user_defined_name
+</pre>
+Indices allow fast access to a table, based on values
+for the indexed columns in the namelist.
+<p>
+Indices can be UNIQUE, meaning that the attributes
+of the index cannot take on the same values in the table
+twice.
+<pre>
+CREATE UNIQUE INDEX user_defined_name 
+  ON user_defined_name ( namelist )
+</pre>
+Unique indices can be used to enforce primary and
+secondary key constraints.  After a UNIQUE index on
+a table is created inserts that attempt to insert
+repeat values for the indexed columns will be rejected.
+
+<h3>Views</h3>
+
+Create view and drop view statements initialize and
+drop views, respectively.
+<pre>
+CREATE VIEW user_defined_name optnamelist 
+  AS select_statement
+
+DROP VIEW user_defined_name
+</pre>
+Views are "derived tables" which are defined
+as stored SELECT statements.  They can be used
+as tables, except that they cannot be directly
+mutated.
+<p>
+It is possible to "implement your own views
+in Python".  Please see remotetest.py, gfintrospect
+and the FAQ for discussion.
+
+<h1>Conditions</h1>
+
+Conditions are truth valued boolean expressions
+formed from basic conditions possibly combined using
+NOT, AND, OR (where NOT has highest precedence and
+OR has lowest precedence) and parentheses.
+<p>
+Basic conditions include simple comparisons
+<pre>
+expression = expression
+expression &lt; expression
+expression &gt; expression
+expression &lt;= expression
+expression &gt;= expression
+expression &lt;&gt; expression
+</pre>
+Variants of the simple comparisons are the quantified
+subquery comparisons
+<pre>
+expression = ANY ( subquery )
+expression = ALL ( subquery )
+</pre>
+(and similarly for the other comparison operators).
+The IN predicate tests membership (like =ANY)
+<pre>
+expression IN ( subquery )
+expression NOT IN ( subquery )
+</pre>
+For all the quantified comparisons and IN the
+subquery must generate a single column table.
+<p>
+Also included are the the BETWEEN and NOT BETWEEN predicates
+<pre>
+expression BETWEEN expression AND expression
+expression NOT BETWEEN expression AND expression
+</pre>
+<p>
+The most general subquery predicate is EXISTS and NOT EXISTS
+which places no restriction on the subquery:
+<pre>
+EXISTS (subquery)
+NOT EXISTS (subquery)
+</pre>
+
+<h1>Expressions</h1>
+
+Expressions occur in conditions (WHERE, HAVING, etc.),
+in UPDATE searched assignments,
+and in the select list of select statements.
+<p>
+Expressions are formed from primary expressions,
+possibly combined using the standard arithmetic operators
+and parentheses with the normal precedence.
+<p>
+Primary expressions include numeric and string literals.
+Numeric literals supported are the Python numeric literals.
+String constants are set off by apostrophies, where two
+apostrophe's in sequence represent an  apostrophy in the
+string:
+<pre>
+'SQL string literals ain''t pretty'
+</pre>
+Column name expressions may be unqualified if they are
+unambiguous, or may be qualified with a table name
+or table alias
+<pre>
+bar
+frequents.bar
+f.bar
+</pre>
+The rules for scoping of column names are not covered
+here.  Column names in subqueries may refer to bindings
+in the query (or queries) that contain the sub-query.
+<p>
+Subquery expressions of form
+<pre>
+( select_statement )
+</pre>
+must produce a single column and single row table.
+<p>
+Aggregate operations are only permitted in the select
+list or in the HAVING condition of SELECT statements
+(including subselects).
+<pre>
+COUNT(*)
+COUNT(expression)
+AVG(expression)
+MAX(expression)
+SUM(expression)
+MIN(expression)
+</pre>
+<em><strong>and also including the non-standard extension MEDIAN
+<pre>
+MEDIAN(expression)
+</pre>
+</strong></em>
+Aggregate operations can be applied to distinct values
+as in
+<pre>
+COUNT(DISTINCT expression)
+</pre>
+The Dynamic expression "?" is a placeholder for a value
+bound at evaluation time (from Python values).  See the
+<a href="gadfly.html">
+API discussions </a>
+(Use) for more details on the use of
+dynamic parameters.
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</body>
+</html>
\ No newline at end of file

lib/python/Products/ZGadflyDA/gadfly/gfclient.py

+
+"""client access to gadfly server. (gfserve.py)
+
+Imported as a module this module provides interfaces
+that remotely access a gadfly database server.
+
+Remote connections: gfclient
+
+   connection = gfclient.gfclient(
+      policy,  # the name of the connection policy ["admin" for admin]
+      port,    # the port number the server is running on
+      password,# the password of the policy
+      [machine]) # (optional) the machine where server is running
+                 # (defaults to localhost)
+
+   methods for gfclient connections:
+      gfclient.checkpoint() checkpoint the server (fails silently
+          if connection is not "admin").
+      gfclient.restart() restart the server (fails silently
+          if connection is not "admin").
+      gfclient.shutdown() shutdown the server (fails silently
+          if connection is not "admin").
+      cursor = gfclient.cursor() returns a cursor on this connection
+
+   methods for cursor objects:
+
+      cursor.execute(statement, dynamic_parameters=None)
+          execute the statement with optional dynamic parameters.
+          Dynamic parameters can be a list of tuples for INSERT
+          VALUES statements, otherwise they must be a tuple
+          of values.
+      cursor.execute_prepared(name, dynamic_parameters=None)
+          execute a named statement configured for this connection
+          policy, with optional dynamic parameters. Dynamic
+          parameters permitted as for execute depending on the
+          statement the name refers to.
+      cursor.fetchall()
+          return results of the last executed statement
+          (None for non-queries, or list of tuples).
+
+See gfstest.py for example usage.
+     
+SCRIPT INTERPRETATION:
+
+Server maintenance utilities
+
+COMMAND LINE:
+   python gfclient.py action port admin_password [machine]
+
+TEST EXAMPLE:
+   python gfclient.py shutdown 2222 admin
+
+   action: one of
+      shutdown:  shut down the server with no checkpoint
+      restart: restart the server (re-read the database and recover)
+      checkpoint: force a database checkpoint now
+   port: the port the server is running on
+   admin_password: the administrative password for the server
+   machine: [optional] the machine the server runs on.
+"""
+
+import gfsocket
+
+def main():
+    import sys
+    try:
+        done=0
+        argv = sys.argv
+        [action, port, admin_password] = argv[1:4]
+        from string import atoi
+        port = atoi(port)
+        if len(argv)>4:
+           machine = argv[4]
+        else:
+           machine = None
+        print action, port, admin_password, machine
+        if action not in ["shutdown", "restart", "checkpoint"]:
+           print "bad action", action
+           print
+           return
+        dosimple(action, port, admin_password, machine)
+        done=1
+    finally:
+        if not done:
+            print __doc__
+
+def dosimple(action, port, pw, machine=None):
+    import socket
+    if machine is None:
+       machine = socket.gethostname()
+    conn = gfclient("admin", port, pw, machine)
+    action = getattr(conn, action)
+    print action()
+
+# copied from gfserve
+# shut down the server (admin policy only)
+#   arguments = ()
+#   shutdown the server with no checkpoint
+SHUTDOWN = "SHUTDOWN"
+
+# restart the server (admin only)
+#   arguments = ()
+#   restart the server (recover)
+#   no checkpoint
+RESTART = "RESTART"
+
+# checkpoint the server (admin only)
+#   arguments = ()
+#   checkpoint the server
+CHECKPOINT = "CHECKPOINT"
+
+# exec prepared statement
+#   arguments = (prepared_name_string, dyn=None)
+#   execute the prepared statement with dynamic args.
+#   autocommit.
+EXECUTE_PREPARED = "EXECUTE_PREPARED"
+
+# exec any statement (only if not disabled)
+#   arguments = (statement_string, dyn=None)
+#   execute the statement with dynamic args.
+#   autocommit.
+EXECUTE_STATEMENT = "EXECUTE_STATEMENT"
+
+class gfclient:
+
+    closed = 0
+    
+    def __init__(self, policy, port, password, machine=None):
+        import socket
+        self.policy = policy
+        self.port = port
+        self.password = password
+        if machine is None:
+            machine = socket.gethostname()
+        self.machine = machine
+        
+    def open_connection(self):
+        import socket
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        #print type(sock), sock
+        sock.connect(self.machine, self.port)
+        return sock
+        
+    def send_action(self, action, arguments, socket):
+        gfsocket.send_certified_action(
+          self.policy, action, arguments, self.password, socket)
+        
+    def checkpoint(self):
+        return self.simple_action(CHECKPOINT)
+    
+    def simple_action(self, action, args=()):
+        """only valid for admin policy: force a server checkpoint"""
+        sock = self.open_connection()
+        self.send_action(action, args, sock)
+        data = gfsocket.recv_data(sock)
+        data = gfsocket.interpret_response(data)
+        return data
+    
+    def restart(self):
+        """only valid for admin policy: force a server restart"""
+        return self.simple_action(RESTART)
+        
+    def shutdown(self):
+        """only valid for admin policy: shut down the server"""
+        return self.simple_action(SHUTDOWN)
+    
+    def close(self):
+        self.closed = 1
+        
+    def commit(self):
+        # right now all actions autocommit
+        pass
+        
+    # cannot rollback, autocommit on success
+    rollback = commit
+    
+    def cursor(self):
+        """return a cursor to this policy"""
+        if self.closed:
+            raise ValueError, "connection is closed"
+        return gfClientCursor(self)
+        
+
+class gfClientCursor:
+
+    statement = None
+    results = None
+    description = None
+
+    def __init__(self, connection):
+        self.connection = connection
+        
+    # should add fetchone fetchmany
+    def fetchall(self):
+        return self.results
+    
+    def execute(self, statement=None, params=None):
+        con = self.connection
+        data = con.simple_action(EXECUTE_STATEMENT, (statement, params))
+        (self.description, self.results) = data
+    
+    def execute_prepared(self, name, params=None):
+        con = self.connection
+        data = con.simple_action(EXECUTE_PREPARED, (name, params))
+        if data is None:
+            self.description = self.results = None
+        else:
+            (self.description, self.results) = data
+    
+    def setoutputsizes(self, *args):
+        pass # not implemented
+        
+    def setinputsizes(self):
+        pass # not implemented
+        
+if __name__=="__main__":
+    main()

lib/python/Products/ZGadflyDA/gadfly/gfinstall.py

+#!/usr/local/bin/python
+
+"""Gadfly installation script.
+
+Build the sql grammar.
+
+usage
+  python <thismodule>
+for a simple install or
+  python <thismodule> force
+for a full rebuild (with grammar regeneration).
+
+In the current directory find or create sql.mar and sqlwhere.py
+where sql.mar has the marshalled grammar data structures
+for parsing sql and sqlwhere.py is a module that indicates
+where the grammar file is as value of sqlwhere.filename.
+"""
+
+marfile = "sql.mar"
+modfile = "sqlwhere.py"
+
+print __doc__
+
+from os import getcwd, path
+cwd = getcwd()
+
+modtemplate ="""
+'''this module indicates where the sql datastructures are marshalled
+   Auto generated on install: better not touch!
+'''
+
+filename = %s
+"""
+
+#wheremod = cwd + "/" + modfile
+#where = cwd + "/" + marfile
+wheremod = path.join(cwd, modfile)
+where = path.join(cwd, marfile)
+print
+print "now creating", wheremod
+f = open(wheremod, "w")
+f.write( modtemplate % (`where`,) )
+f.close()
+
+from sqlgen import BuildSQL, getSQL
+import sys
+argv = sys.argv
+force = 0
+#print argv
+if len(argv)>1 and argv[1]=="force":
+   force = 1
+if not force:
+   try:
+       sql = getSQL()
+   except:
+       print "exception", sys.exc_type, sys.exc_value
+       print "during load of SQL grammar structures."
+       print "Apparently the SQL grammar requires regeneration"
+       force = 1
+if force:
+   print "now generating parser structures (this might take a while)..."
+   #where = cwd + "/" + marfile
+   print "building in", where
+   sql = BuildSQL(where)
+print
+print "done."
+

lib/python/Products/ZGadflyDA/gadfly/gfintrospect.py

+"""
+View based introspection and extension views
+"""
+
+import gfdb0
+
+class RemoteView(gfdb0.View):
+
+    """Virtual superclass.  See text for methods and members to define."""
+    
+    # Usually redefine __init__
+    def __init__(self):
+        pass
+    
+    # set static (static=1) or dynamic (static=0)
+    # for static tuple seq is generated once per load
+    # for non-static tuple seq is regenerated once per query
+    #  which uses the view.
+    static = 0
+    
+    # define the column_names
+    column_names = ['Column1']
+    
+    # define the row generator
+    def listing(self):
+        """return list of values (1 column)
+           or list of tuples of values (>1 column).
+           size of elts should match number of columns."""
+        return [0]
+        
+    # possibly redefine __repr__ and irepr
+    def __repr__(self):
+        return "<Remote View %s at %s>" % (self.__class__, id(self))
+        
+    irepr = __repr__
+    
+    # for introspective methods possibly redefine relbind
+    def relbind(self, db, atts):
+        return self
+        
+    ### don't touch the following unless you are a guru!
+    cached_rows = None
+    
+    def uncache(self):
+        if self.static: return
+        self.cached_rows = None
+        
+    def attributes(self):
+        from string import upper
+        return map(upper, self.column_names)
+        
+    def rows(self, andseqs=0):
+        cached_rows = self.cached_rows
+        if cached_rows is None:
+            tups = list(self.listing())
+            from sqlsem import kjbuckets
+            undump = kjbuckets.kjUndump
+            attributes = tuple(self.attributes())
+            for i in xrange(len(tups)):
+                tups[i] = undump(attributes, tups[i])
+            cached_rows = self.cached_rows = tups
+        tups = cached_rows[:]
+        if andseqs:
+            return (tups, range(len(tups)))
+        else:
+            return tups
+
+class DualView(RemoteView):
+    """static one row one column view for testing.
+       (Inspired by Oracle DUAL relation)."""
+    # trivial example extension view
+    
+    static = 1
+    
+    column_names = ['Column1']
+    
+    def listing(self):
+        return [0]
+        
+class DictKeyValueView(RemoteView):
+    """Less trivial example. Dict keys/values converted to strings"""
+    
+    static = 0 # regenerate in case dict changes
+    
+    column_names = ["key", "value"]
+    
+    mapstring = 1
+    
+    def __init__(self, dict=None):
+        if dict is None: dict = {}
+        self.dict = dict
+        
+    def listing(self):
+        items = self.dict.items()
+        if self.mapstring:
+            def mapper(item):
+                return tuple(map(str, item))
+            return map(mapper, items)
+        else:
+            return items
+        
+class RelationsView(DictKeyValueView):
+    """list of relations and whether they are views."""
+    
+    column_names = ["table_name", "is_view"]
+    mapstring = 0
+    
+    def relbind(self, db, atts):
+        rels = db.rels
+        dict = {}
+        for relname in rels.keys():
+            dict[relname] = rels[relname].is_view
+        self.dict = dict
+        return self
+        
+class IndicesView(DictKeyValueView):
+    """list of indices and relations they index"""
+    
+    column_names = ["index_name", "table_name", "is_unique"]
+    
+    mapstring = 0
+    
+    def relbind(self, db, atts):
+        rels = db.rels
+        dict = {}
+        for relname in rels.keys():
+            rel = rels[relname]
+            if not rel.is_view:
+                index_list = rels[relname].index_list
+                for index in index_list:
+                    dict[index.name] = (relname, index.unique)
+        self.dict = dict
+        return self
+        
+    def listing(self):
+        L = []
+        dict = self.dict
+        keys = dict.keys()
+        for k in keys:
+            L.append( (k,) + dict[k] )
+        return L
+        
+class DataDefsView(DictKeyValueView):
+    """Data defs (of non-special views) and definition dumps."""
+    
+    column_names = ["name", "defn"]
+    
+    mapstring = 1
+    
+    def relbind(self, db, atts):
+        self.dict = db.datadefs
+        return self
+        
+class ColumnsView(RemoteView):
+    """table_names and columns therein."""
+    
+    column_names = ["table_name", "column_name"]
+    
+    def relbind(self, db, atts):
+        rels = db.rels
+        pairs = []
+        for relname in rels.keys():
+            for att in rels[relname].attributes():
+                pairs.append( (relname, att) )
+        self.pairs = pairs
+        return self
+        
+    def listing(self):
+        return self.pairs
+        
+class IndexAttsView(ColumnsView):
+    """indices and attributes."""
+    
+    column_names = ["index_name", "column_name"]
+    
+    def relbind(self, db, atts):
+        indices = db.indices
+        pairs = []
+        for iname in indices.keys():
+            for att in indices[iname].attributes():
+                pairs.append( (iname, att) )
+        self.pairs = pairs
+        return self
+        

lib/python/Products/ZGadflyDA/gadfly/gfrecover.html

+<HTML>
+<HEAD>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=windows-1252">
+<META NAME="Generator" CONTENT="Microsoft Word 97">
+<TITLE>Gadfly: recovery</TITLE>
+<META NAME="Template" CONTENT="C:\PROGRAM FILES\MICROSOFT OFFICE\OFFICE\html.dot">
+</HEAD>
+<BODY BGCOLOR="#ffffff">
+
+<table><tr><td>
+<img src="gadfly.JPG">
+</td><td>
+<H1>Gadfly Recovery</H1>
+</td></tr></table>
+
+<P>In the event of a software glitch or crash Gadfly may terminate without having stored committed updates. 
+A recovery strategy attempts to make sure
+that the unapplied commited updates are applied when the database restarts. 
+It is always assumed that there is only one primary (server) process controlling the database (possibly with 
+multiple clients). </P>
+
+<P>Gadfly uses a simple LOG with deferred updates recovery mechanism. Recovery should be possible in the 
+presence of non-disk failures (server crash, system crash). Recovery after a disk crash is not available 
+for Gadfly as yet, sorry. </P>
+
+<P>Due to portability problems Gadfly does not prevent multiple processes from "controlling" the database at 
+once. For read only access multiple instances are not a problem, but for access with modification, the processes 
+may collide and corrupt the database. For a read-write database, make sure only one (server) process controls 
+the database at any given time. </P>
+
+<P>The only concurrency control mechanism that provides serializability for Gadfly as yet is the trivial one -- 
+the server serves all clients serially. This will likely change for some variant of the system at some point. </P>
+
+<P>This section explains the basic recovery mechanism. </P>
+
+<H1>Normal operation</H1>
+
+<H3>Precommit</H3>
+<P>During normal operations any active tables are in memory in the process. 
+Uncommitted updates for a transaction are kept in "shadow tables" until the transaction commits using
+<pre>
+   connection.commit()
+</pre>
+The shadow tables remember the mutations that have been applied to them. The permanent table copies 
+are only modified after commit time.  A commit commits all updates for all cursors for the connection.
+Unless the autocommit feature is disabled (see below) a
+commit normally always triggers a checkpoint too.</P>
+
+A rollback
+<pre>
+   connection.rollback()
+</pre>
+explicitly discards all uncommitted updates and restores the connection to the previously
+committed state.</p>
+
+<P>There is a 3rd level of shadowing for statement sequences executed by a cursor.
+In particular the design attempts to make sure that if 
+<pre>
+  cursor.execute(statement)
+</pre>
+fails with an error, then the shadow database will contain no updates from
+the partially executed statement (which may be a sequence of statements)
+but will reflect other completed updates that may have not been committed.
+
+<H3>Commit</H3>
+
+<P>At commit, operations applied to shadow tables are written 
+out in order of application to a log file before being permanently 
+applied to the active database. Finally a commit record is written to 
+the log and the log is flushed. At this point the transaction is considered 
+committed and recoverable, and a new transaction begins.
+Finally the values of the shadow tables replace 
+the values of the permanent tables in the active database,
+(but not in the database disk files until checkpoint, if autocheckpoint
+is disabled). </P>
+
+<H3>Checkpoint</H3>
+<P>A checkpoint operation brings the persistent copies of the tables on 
+disk in sync with the in-memory copies in the active database. Checkpoints 
+occur at server shut down or periodically during server operation. 
+The checkpoint operation runs in isolation (with no database access 
+allowed during checkpoint). </P>
+
+<p><em>Note: database connections normally run a checkpoint
+after every commit, unless you set
+<pre>
+    connection.autocheckpoint = 0
+</pre>
+which asks that checkpoints be done explicitly by the program using
+<pre>
+    connection.commit() # if appropriate
+    connection.checkpoint()
+</pre>
+Explicit checkpoints should make the database perform better,
+since the disk files are written less frequently, but
+in order to prevent unneeded (possibly time consuming)
+recovery operations after a database
+is shutdown and restarted it is important to always execute an explicit
+checkpoint at server shutdown, and periodically during long server
+runs.</em></p>
+
+<p><strong>Note that if any outstanding operations are uncommitted
+at the time of a checkpoint (when autocheckpoint is disabled) the
+updates will be lost (ie, it is equivalent to a rollback).
+</strong></p>
+
+<P>At checkpoint the old persistent value of each table that has been updated since 
+the last checkpoint is copied to a back up file, and the currently active value is 
+written to the permanent table file.  Finally if the data definitions have changed
+the old definitions are stored to a backup file and the new definitions are written
+to the permanent data definition file.  To signal successful checkpoint the
+log file is then deleted.</P>
+<P>
+ At this point (after log deletion) the database is considered 
+quiescent (no recovery required). Finally all back up table files are deleted. 
+[Note, it might be good to keep old logs around... Comments?] </P>
+
+<P>Each table file representation is annotated with a checksum, 
+so the recovery system can check that the file was stored correctly. </P>
+
+<H1>Recovery</H1>
+<P>When a database restarts it automatically determines whether 
+the last active instance shut down normally and whether recovery 
+is required. Gadfly discovers the need for recovery by detecting 
+a non-empty current log file. </P>
+
+<P>To recover the system Gadfly first scans the log file to determine committed transactions. 
+Then Gadfly rescans the log file applying the operations of committed 
+transactions to the in memory table values in the order recorded. 
+When reading in table values for the purpose of recovery Gadfly looks 
+for a backup file for the table first. If the backup is not corrupt, 
+its value is used, otherwise the permanent table file is used. </P>
+<P>After recovery Gadfly runs a normal checkpoint before resuming 
+normal operation. </P>
+<p>
+<strong>
+Please note: Although I have attempted to provide a robust
+implementation
+for this software I do not guarantee its correctness.  I hope
+it will work well for you but I do not assume any legal
+responsibility for problems anyone may have during use
+of these programs.
+</strong>
+
+<p>
+<a href="mailto:arw@ifu.net">feedback</a><br>
+<a href="../index.html">home</a><br>
+<a href="index.html">Gadfly home</a>
+</BODY>
+</HTML>

lib/python/Products/ZGadflyDA/gadfly/gfsocket.py

+
+
+"""socket interactions for gadfly client and server"""
+
+from select import select
+
+# responses
+
+SUCCESS = "SUCCESS"
+EXCEPTION = "EXCEPTION"
+            
+def reply_exception(exception, info, socket):
+    """send an exception back to the client"""
+    # any error is invisible to client
+    from gfserve import ServerError
+    import sys
+    try:
+        reply( (EXCEPTION, (exception, info)), socket)
+    except:
+        #info = "%s %s" % (sys.exc_type, sys.exc_value)
+        socket.close()
+        #raise ServerError, "reply_exception failed: "+`info`
+        
+def reply_success(data, socket):
+    """report success with data back to client"""
+    reply( (SUCCESS, data), socket)
+        
+def reply(data, socket):
+    from marshal import dumps
+    marshaldata = dumps(data)
+    send_packet(socket, marshaldata)
+    socket.close()
+    
+def send_packet(socket, data):
+    """blast out a length marked packet"""
+    send_len(data, socket)
+    socket.send(data)
+    
+def send_len(data, socket):
+    """send length of data as cr terminated int rep"""
+    info = `len(data)`+"\n"
+    socket.send(info)
+    
+def send_certified_action(actor_name, action, arguments, password, socket):
+    from marshal import dumps
+    marshaldata = dumps( (action, arguments) )
+    cert = certificate(marshaldata, password)
+    #print actor_name, cert,  marshaldata
+    marshaldata = dumps( (actor_name, cert, marshaldata) )
+    send_packet(socket, marshaldata)
+    
+def unpack_certified_data(data):
+    from marshal import loads
+    # sanity check
+    unpack = (actor_name, certificate, marshaldata) = loads(data)
+    return unpack
+    
+def recv_data(socket, timeout=10):
+    """receive data or time out"""
+    from time import time
+    endtime = time() + timeout
+    reader = Packet_Reader(socket)
+    done = 0
+    while not done:
+        timeout = endtime - time()
+        if timeout<0:
+            raise IOError, "socket time out (1)"
+        (readable, dummy, error) = select([socket], [], [socket], timeout)
+        if error:
+            raise IOError, "socket in error state"
+        if not readable:
+            raise IOError, "socket time out (2)"
+        reader.poll()
+        done = (reader.mode==READY)
+    return reader.data
+    
+def interpret_response(data):
+    """interpret response data, raise exception if needed"""
+    from marshal import loads
+    (indicator, data) = loads(data)
+    if indicator==SUCCESS:
+        return data
+    elif indicator==EXCEPTION:
+        # ???
+        raise EXCEPTION, data
+    else:
+        raise ValueError, "unknown indicator: "+`indicator`
+    
+# packet reader modes
+LEN = "LEN"
+DATA = "DATA"
+READY = "READY"
+ERROR = "ERROR"
+
+BLOCK_SIZE = 4028
+
+LEN_LIMIT = BLOCK_SIZE * 10
+    
+class Packet_Reader:
+    """nonblocking pseudo-packet reader."""
+    
+    # packets come in as decimal_len\ndata
+    # (note: cr! not crlf)
+    
+    # kick too large requests if set
+    limit_len = LEN_LIMIT
+    
+    def __init__(self, socket):
+        self.socket = socket
+        self.length = None
+        self.length_remaining = None
+        self.len_list = []
+        self.data_list = []
+        self.received = ""
+        self.data = None
+        self.mode = LEN
+        
+    def __len__(self):
+        if self.mode is LEN:
+            raise ValueError, "still reading length"
+        return self.length
+        
+    def get_data(self):
+        if self.mode is not READY:
+            raise ValueError, "still reading"
+        return self.data
+        
+    def poll(self):
+        mode = self.mode
+        if mode is READY:
+            raise ValueError, "data is ready"
+        if mode is ERROR:
+            raise ValueError, "socket error previously detected"
+        socket = self.socket
+        (readable, dummy, error) = select([socket], [], [socket], 0)
+        if error:
+            self.socket.close()
+            self.mode = ERROR
+            raise ValueError, "socket is in error state"
+        if readable:
+            if mode is LEN:
+                self.read_len()
+            # note: do not fall thru automatically
+            elif mode is DATA:
+                self.read_data()
+                
+    def read_len(self):
+        """assume socket is readable now, read length"""
+        socket = self.socket
+        received = self.received
+        len_list = self.len_list
+        if not received:
+            # 10 bytes at a time until len is read.
+            received = socket.recv(10)
+        while received:
+            # consume, test one char
+            input = received[0]
+            received = received[1:]
+            if input == "\n":
+                # done reading length
+                from string import join, atoi
+                try:
+                    length = self.length = atoi(join(len_list, ""))
+                except:
+                    self.mode = ERROR
+                    socket.close()
+                    raise ValueError, "bad len string? "+`len_list`
+                self.received = received
+                self.length_remaining = length
+                self.mode = DATA
+                limit_len = self.limit_len
+                if limit_len and length>limit_len:
+                    raise ValueError, "Length too big: "+`(length, limit_len)`
+                return
+            if len(len_list)>10:
+                self.mode = ERROR
+                socket.close()
+                raise ValueError, "len_list too long: "+`len_list`
+            len_list.append(input)
+            if not received:
+                (readable, dummy, error) = select(\
+                   [socket], [], [socket], 0)
+                if error:
+                    self.mode = ERROR
+                    socket.close()
+                    raise ValueError, "socket in error state"
+                if readable:
+                    received = socket.recv(10)
+        # remember extra data received.
+        self.received = received
+
+    def read_data(self):
+        # assume socket is readable
+        socket = self.socket
+        received = self.received
+        length_remaining = self.length_remaining
+        data_list = self.data_list
+        if received:
+            data_list.append(received)
+            self.received = ""
+            length_remaining = length_remaining - len(received)
+        recv_len = max(length_remaining, BLOCK_SIZE)
+        received = socket.recv(recv_len)
+        if received:
+            data_list.append(received)
+            length_remaining = length_remaining - len(received)
+        if length_remaining<1:
+            self.mode = READY
+            from string import join
+            self.data = join(data_list, "")
+        self.length_remaining = length_remaining
+
+def certificate(String, password):
+    """generate a certificate for a string, using a password"""
+    from md5 import new
+    if not String:
+        raise ValueError, "cannot generate certificate for empty string"
+    taggedstring = password + String
+    return new(taggedstring).digest()
+    
+def certify(String, cert, password):
+    """check a certificate for a string"""
+    return certificate(String, password) == cert
+

lib/python/Products/ZGadflyDA/gadfly/gfstest.py

+
+"""test script for gadfly client and server
+
+Usage:  This script interacts with the test database generated
+  by gftest.py.  To start the server from the directory containing
+  the dbtest directory use:
+
+     python gfstest.py start
+
+  THIS WILL ONLY WORK IF YOU CREATED THE test DATABASE IN
+  DIRECTORY dbtest FIRST USING 
+
+     python gftest.py dbtest
+
+  UNLESS YOU RUN THE SERVER IN THE BACKGROUND THE SERVER WILL
+  HOG THE WINDOW YOU STARTED IT IN AND YOU WILL HAVE TO USE ANOTHER
+  WINDOW UNTIL THE SERVER IS SHUT DOWN (SEE BELOW).
+
+  Then from *anywhere* (on the same machine) access the database
+  using
+     python gfstest.py restart
+       - restart the server (reread the database)
+     python gfstest.py checkpoint
+       - force checkpoint the server
+     python gfstest.py queries
+       - run some example queries and updates
+     python gfstest.py policy_test
+       - test the policies test and test1 created by the startup
+         function in this module.
+     python gfstest.py bogusshutdown
+       - attempt to shut down the server with a bogus password
+         [should generate an exception]
+
+...and finally
+     python gfstest.py shutdown
+       - shut down the server for real.
+
+As mentioned the startup function of this module illustrates
+how to create a "startup" function for a server and initialize policy
+objects with named, prepared queries.
+
+"""
+
+PORT = 2222
+DB = "test"
+DBDIR = "dbtest"
+PW = "admin"
+STARTUP = "gfstest"
+
+import sys, socket
+
+
+def main():
+    argv = sys.argv
+    command = argv[1]
+    if command=="start":
+        print "attempting to start the server"
+        from gfserve import Server
+        print "making a server on", PORT, DB, DBDIR, PW, STARTUP
+        S = Server(PORT, DB, DBDIR, PW, STARTUP)
+        print "initializing the server"
+        S.init()
+        print "starting the server", S.connection
+        S.start()
+    elif command=="shutdown":
+        dosimple("shutdown", PW)
+    elif command=="bogusshutdown":
+        print "BOGUS shutdown attempt"
+        dosimple("shutdown", "bad password")
+    elif command=="restart":
+        dosimple("restart", PW)
+    elif command=="checkpoint":
+        dosimple("checkpoint", PW)
+    elif command=="queries":
+        doqueries()
+    elif command=="policy_test":
+        policy_test()
+    else:
+        print "unknown command", command
+        print __doc__
+
+def policy_test():
+    """test the test1 and test policies"""
+    print "testing non-admin policies test and test1"
+    from gfclient import gfclient
+    import sys
+    machine = socket.gethostname()
+    conn = gfclient("test", PORT, "test", machine)
+    cursor = conn.cursor()
+    print "testing test policy: nan values before:"
+    cursor.execute_prepared("getnan")
+    for x in cursor.fetchall():
+        print x
+    print "updating nan"
+    cursor.execute_prepared("updatenan", ("pabst", 4))
+    print "nan after"
+    cursor.execute_prepared("getnan")
+    for x in cursor.fetchall():
+        print x
+    print "updating nan again"
+    cursor.execute_prepared("updatenan", ("rollingrock", 1))
+    print "trying an illegal update"
+    try:
+        cursor.execute("delete from frequents")
+    except:
+        print "exception", sys.exc_type, sys.exc_value
+        print "as expected"
+    else:
+        raise "DAMN!", "illegal query apparently completed!!!"
+    print; print "testing policy test1"; print
+    conn = gfclient("test1", PORT, "test1", machine)
+    cursor = conn.cursor()
+    print "getting norm"
+    cursor.execute_prepared("qlike", ("norm",))
+    print cursor.description
+    for x in cursor.fetchall():
+        print x
+    print "trying an illegal query again"
+    try:
+        cursor.execute("create table test(name varchar)")
+    except:
+        print "exception", sys.exc_type, sys.exc_value
+        print "as expected"
+    else:
+        raise "Damn!(2)", "illegal query apparently completed"
+
+def startup(admin_policy, connection, Server_instance):
+    """example startup script.
+
+       add a policies test and test1 passwords same
+         test1 is allowed to query the frequents table by name
+         test is allowed to update likes where drinker='nan'
+       also add prepared query dumpwork to admin_policy.
+    """
+    from gfserve import Policy
+    admin_policy["dumpwork"] = "select * from work"
+    test1 = Policy("test1", "test1", connection, queries=0)
+    test = Policy("test", "test", connection, queries=0)
+    test1["qlike"] = "select * from likes where drinker=?"
+    test["updatenan"] = """
+      update likes
+      set beer=?, perday=?
+      where drinker='nan'
+    """
+    test["getnan"] = """
+    select * from likes where drinker='nan'
+    """
+    return {"test": test, "test1": test1}
+
+def doqueries():
+    print "executing queries and updates"
+    from gfclient import gfclient
+    import sys
+    machine = socket.gethostname()
+    conn = gfclient("admin", PORT, PW, machine)
+    cursor = conn.cursor()
+    for q in admin_queries:
+        print;print;print q;print
+        try:
+            cursor.execute(q)
+        except:
+            print "exception in execute"
+            print sys.exc_type
+            v = sys.exc_value
+            from types import TupleType, ListType
+            if type(v) in (TupleType, ListType):
+               for x in v: print x
+            else:
+               print v
+        else:
+            print "executed"
+            #print q
+            print "description"
+            print cursor.description
+            print "results"
+            try:
+                r = cursor.fetchall()
+                if r is None:
+                    print "no results"
+                else:
+                    for x in r:
+                        print x
+            except:
+                print "exception in results"
+                print sys.exc_type, sys.exc_value
+                print dir(cursor)
+    # try dumpwork
+    print; print; print "dumpwork"; print
+    cursor.execute_prepared("dumpwork")
+    for x in cursor.fetchall():
+        print x
+    # try dynamic parameters
+    stat = """
+    select distinct drinker
+    from likes l, serves s
+    where l.beer = s.beer and s.bar=?
+    """
+    print; print stat; print "dynamic query ?=cheers"
+    cursor.execute(stat, ("cheers",))
+    for x in cursor.fetchall():
+        print x
+            
+admin_queries = [
+"""select count(*) from work""",
+"""select * from frequents""",
+"""select count(*) from frequents""",
+"""select count(drinker) from frequents""",
+"""insert into frequents(drinker, bar, perweek)
+     values ('sally', 'cheers', 2)""",
+"""select * from frequents""",
+"""select syntax error from work""",
+"""select drinker, count(bar) from frequents
+   group by drinker""",
+]
+
+def dosimple(command, pw):
+    print "attempting remote %s (%s) for server on local machine" % (command, pw)
+    from gfclient import gfclient
+    machine = socket.gethostname()
+    conn = gfclient("admin", PORT, pw, machine)
+    action = getattr(conn, command)
+    print action()
+
+if __name__=="__main__":
+    main()
\ No newline at end of file

lib/python/Products/ZGadflyDA/gadfly/sqlgtest.py

+"test parses for sql grammar"
+
+test = [
+"select a from x where b=c",
+"select distinct x.a from x where x.b=c",
+"select all a from x where b=c",
+"select a from x, y where b=c or x.d=45",
+"select a as k from x d, y as m where b=c",
+"select 1 as n, a from x where b=c",
+"select * from x",
+"select a from x where b=c",
+"select a from x where not b=c or d=1 and e=5",
+"select a from x where a=1 and (x.b=3 or not b=c)",
+"select -1 from x",
+"select -1e6j from x",
+"insert into table1 (a,b,c) values (-1e6+3j, -34e10, 56j)"
+]
\ No newline at end of file

lib/python/Products/ZGadflyDA/gadfly/sqlwhere.py

+
+'''this module indicates where the sql datastructures are marshalled
+   Auto generated on install: better not touch!
+'''