casacore
TableParse.h
Go to the documentation of this file.
1 //# TableParse.h: Classes to hold results from table grammar parser
2 //# Copyright (C) 1994,1995,1997,1998,1999,2000,2001,2003
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_TABLEPARSE_H
29 #define TABLES_TABLEPARSE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/tables/Tables/Table.h>
34 #include <casacore/tables/Tables/TableDesc.h>
35 #include <casacore/tables/TaQL/ExprNode.h>
36 #include <casacore/tables/TaQL/TaQLResult.h>
37 #include <casacore/tables/TaQL/ExprGroup.h>
38 #include <casacore/casa/BasicSL/String.h>
39 #include <casacore/casa/Utilities/Sort.h>
40 #include <casacore/casa/Containers/Record.h>
41 #include <casacore/casa/Containers/Block.h>
42 #include <map>
43 #include <vector>
44 #include <limits>
45 
46 namespace casacore { //# NAMESPACE CASACORE - BEGIN
47 
48 //# Forward Declarations
49 class TableExprNodeSet;
50 class TableExprNodeSetElem;
51 class TableExprNodeIndex;
52 class TableColumn;
53 class AipsIO;
54 template<class T> class Vector;
55 template<class T> class ArrayColumn;
56 
57 
58 // <summary>
59 // Class to hold values from table grammar parser
60 // </summary>
61 
62 // <use visibility=local>
63 
64 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tTableGram">
65 // </reviewed>
66 
67 // <prerequisite>
68 //# Classes you should understand before using this one.
69 // </prerequisite>
70 
71 // <etymology>
72 // TableParse is the class used to parse a table command.
73 // </etymology>
74 
75 // <synopsis>
76 // TableParse is used by the parser of table select statements.
77 // The parser is written in Bison and Flex in files TableGram.y and .l.
78 // The statements in there use the routines in this file to act
79 // upon a reduced rule.
80 // Since multiple tables can be given (with a shorthand), the table
81 // names are stored in a container. The variable names can be qualified
82 // by the table name and will be looked up in the appropriate table.
83 //
84 // A select command is similar to SQL and can look like:
85 // SELECT columns FROM tab1 sh1, tab2 sh2, tab3 WHERE
86 // sh1.field == 3*sh1.field2 ... ORDERBY columns GIVING table
87 // This is described in more detail in TableGram.l.
88 //
89 // The class TableParse only contains information about a table
90 // used in the table command.
91 //
92 // Global functions are used to operate on the information.
93 // The main function is the global function tableCommand.
94 // It executes the given TaQL command and returns the resulting table.
95 // This is, in fact, the only function to be used by a user.
96 // </synopsis>
97 
98 // <motivation>
99 // It is necessary to be able to give a table select command in ASCII.
100 // This can be used in a CLI or in the table browser to get a subset
101 // of a table or to sort a table.
102 // </motivation>
103 
104 //# <todo asof="$DATE:$">
105 //# A List of bugs, limitations, extensions or planned refinements.
106 //# </todo>
107 
108 
110 {
111 
112 public:
113  // Default constructor for container class.
114  TableParse();
115 
116  // Copy constructor (copy semantics).
117  TableParse (const TableParse&);
118 
119  // Assignment (copy semantics).
121 
122  // Associate the table and the shorthand.
123  TableParse (const Table& table, const String& shorthand);
124 
125  // Test if shorthand matches.
126  Bool test (const String& shortHand) const;
127 
128  // Get the shorthand.
129  const String& shorthand() const;
130 
131  // Get table object.
132  const Table& table() const;
133 
134 private:
137 };
138 
139 
140 
141 // <synopsis>
142 // Parse and execute the given command.
143 // It will open (and close) all tables needed.
144 // It returns the resulting table.
145 // The command type (select or update) and the selected or updated
146 // column names can be returned.
147 // Zero or more temporary tables can be used in the command
148 // using the $nnn syntax.
149 // </synopsis>
150 // <group name=tableCommand>
151 TaQLResult tableCommand (const String& command);
152 
153 TaQLResult tableCommand (const String& command,
154  const Table& tempTable);
155 TaQLResult tableCommand (const String& command,
156  const std::vector<const Table*>& tempTables);
157 TaQLResult tableCommand (const String& command,
158  Vector<String>& columnNames);
159 TaQLResult tableCommand (const String& command,
160  Vector<String>& columnNames,
161  String& commandType);
162 TaQLResult tableCommand (const String& command,
163  const std::vector<const Table*>& tempTables,
164  Vector<String>& columnNames);
165 TaQLResult tableCommand (const String& command,
166  const std::vector<const Table*>& tempTables,
167  Vector<String>& columnNames,
168  String& commandType);
169 // </group>
170 
171 
172 
173 
174 // <summary>
175 // Helper class for sort keys in TableParse
176 // </summary>
177 
178 // <use visibility=local>
179 
180 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
181 // </reviewed>
182 
183 // <prerequisite>
184 //# Classes you should understand before using this one.
185 // <li> TableParse
186 // </prerequisite>
187 
188 // <etymology>
189 // TableParseSort holds a sort expression and order.
190 // </etymology>
191 
192 // <synopsis>
193 // A table command is parsed.
194 // An object of this class is used to hold the sort expression
195 // and sort order.
196 // </synopsis>
197 
198 
200 {
201 public:
202  // Construct from a given expression.
203  // The order is not given.
204  TableParseSort();
205 
206  // Construct from a given expression.
207  // The order is not given.
208  explicit TableParseSort (const TableExprNode&);
209 
210  // Construct from a given expression and for the given order.
212 
213  ~TableParseSort();
214 
215  // Get the expression node.
216  const TableExprNode& node() const;
217 
218  // Get the sort order.
219  Sort::Order order() const;
220 
221  // Is the order given?
222  Bool orderGiven() const;
223 
224 private:
225  // Check if the node results in a scalar and does not contain
226  // aggregate functions.
227  void checkNode() const;
228 
232 };
233 
234 
235 
236 
237 // <summary>
238 // Helper class for updates in TableParse
239 // </summary>
240 
241 // <use visibility=local>
242 
243 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
244 // </reviewed>
245 
246 // <prerequisite>
247 //# Classes you should understand before using this one.
248 // <li> TableParse
249 // </prerequisite>
250 
251 // <etymology>
252 // TableParseUpdate holds a column name, optional indices, optional mask,
253 // and an update expression.
254 // </etymology>
255 
256 // <synopsis>
257 // A table command is parsed.
258 // An object of this class is used to hold the column name, optional indices,
259 // and value expression for the UPDATE command.
260 // </synopsis>
261 
262 
264 {
265 public:
267  : indexPtr_p(0) {}
268 
269  // Construct from a column name and expression.
270  // By default it checks if no aggregate functions are used.
271  TableParseUpdate (const String& columnName,
272  const String& columnNameMask,
273  const TableExprNode&,
274  Bool checkAggr=True);
275 
276  // Construct from a column name, subscripts or mask, and expression.
277  // It checks if no aggregate functions are used.
278  TableParseUpdate (const String& columnName,
279  const String& columnNameMask,
280  const TableExprNodeSet& indices,
281  const TableExprNode&,
282  const TaQLStyle&);
283 
284  // Construct from a column name, subscripts and mask, and expression.
285  // It checks if no aggregate functions are used.
286  // It checks if one of the indices represents subscripts, the other a mask.
287  TableParseUpdate (const String& columnName,
288  const String& columnNameMask,
289  const TableExprNodeSet& indices1,
290  const TableExprNodeSet& indices2,
291  const TableExprNode&,
292  const TaQLStyle&);
293  // Handle the subscripts or mask.
294  // It checks if subscripts or mask was not already used.
295  void handleIndices (const TableExprNodeSet& indices,
296  const TaQLStyle& style);
297  ~TableParseUpdate();
298 
299  // Set the column name.
300  void setColumnName (const String& name);
301 
302  // Set the column name forthe mask.
303  void setColumnNameMask (const String& name);
304 
305  // Get the column name.
306  const String& columnName() const;
307 
308  // Get the possible column name for the mask.
309  const String& columnNameMask() const;
310 
311  // Tell if the mask is given first (i.e., before slice).
312  Bool maskFirst() const
313  { return maskFirst_p; }
314 
315  // Get the pointer to the indices.
316  TableExprNodeIndex* indexPtr() const;
317 
318  // Get the index expression node.
319  const TableExprNode& indexNode() const;
320 
321  // Get the expression node.
322  // <group>
323  const TableExprNode& node() const;
324  TableExprNode& node();
325  // </group>
326 
327  // Get the mask.
328  const TableExprNode& mask() const
329  { return mask_p; }
330 
331  // Adapt the possible unit of the expression to the possible unit
332  // of the column.
333  void adaptUnit (const Unit& columnUnit);
334 
335 private:
338  Bool maskFirst_p; //# True = mask is given before slice
339  TableExprNodeIndex* indexPtr_p; //# copy of pointer in indexNode_p
343 };
344 
345 
346 
347 
348 // <summary>
349 // Select-class for flex/bison scanner/parser for TableParse
350 // </summary>
351 
352 // <use visibility=local>
353 
354 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
355 // </reviewed>
356 
357 // <prerequisite>
358 //# Classes you should understand before using this one.
359 // <li> TableParse
360 // <li> TableGram.l and .y (flex and bison grammar)
361 // </prerequisite>
362 
363 // <synopsis>
364 // This class is needed for the the actions in the flex scanner
365 // and bison parser.
366 // This stores the information by constructing TableParse objects
367 // as needed and storing them in a vector.
368 // </synopsis>
369 
370 // <motivation>
371 // It is necessary to be able to give a table select command in ASCII.
372 // This can be used in a CLI or in the table browser to get a subset
373 // of a table or to sort a table.
374 // </motivation>
375 
376 //# <todo asof="$DATE:$">
377 //# A List of bugs, limitations, extensions or planned refinements.
378 //# </todo>
379 
380 
382 {
383 public:
384  enum CommandType {
393  PSHOW
394  };
395 
397  GROUPBY=1,
398  AGGR_FUNCS=2,
399  ONLY_COUNTALL=4
400  };
401 
402  // Construct.
404 
405  // Destructor.
406  ~TableParseSelect();
407 
408  // Return the command type.
410  { return commandType_p; }
411 
412  // Return the expression node.
414  { return node_p; }
415 
416  // Create a temporary table in no tables are given in FROM.
417  void makeTableNoFrom (const vector<TableParseSelect*>& stack);
418 
419  // Execute the select command (select/sort/projection/groupby/having/giving).
420  // The setInGiving flag tells if a set in the GIVING part is allowed.
421  // The mustSelect flag tells if a SELECT command must do something.
422  // Usually that is required, but not for a SELECT in an INSERT command.
423  // Optionally the maximum nr of rows to be selected can be given.
424  // It will be used as the default value for the LIMIT clause.
425  // 0 = no maximum.
426  void execute (Bool showTimings, Bool setInGiving,
427  Bool mustSelect, uInt maxRow, Bool doTracing=False);
428 
429  // Execute a query in a from clause resulting in a Table.
430  Table doFromQuery (Bool showTimings);
431 
432  // Execute a subquery and create an appropriate node for the result.
433  TableExprNode doSubQuery (Bool showTimings);
434 
435  // Test if a subquery has sufficient elements.
436  // It uses default LIMIT=1, but that can be overidden in the subquery.
437  // The flag tells if NOT EXISTS or EXISTS was given.
438  TableExprNode doExists (Bool noexists, Bool showTimings);
439 
440  // Show the expression tree.
441  void show (ostream& os) const;
442 
443  // Keep the selection expression.
444  void handleWhere (const TableExprNode&);
445 
446  // Keep the groupby expressions.
447  // It checks if they are all scalar expressions.
448  void handleGroupby (const vector<TableExprNode>&, Bool rollup);
449 
450  // Keep the having expression.
451  void handleHaving (const TableExprNode&);
452 
453  // Keep the expression of a calculate command.
454  void handleCalcComm (const TableExprNode&);
455 
456  // Keep the create table command.
457  void handleCreTab (const Record& dmInfo);
458 
459  // Keep the column specification in a create table command.
460  void handleColSpec (const String& columnName, const String& dataType,
461  const Record& spec, Bool isCOrder=False);
462 
463  // Reopen the table (for update) used in the ALTER TABLE command.
464  void handleAltTab();
465 
466  // Add columns to the table of ALTER TABLE.
467  // The column descriptions have already been added to tableDesc_p.
468  void handleAddCol (const Record& dmInfo);
469 
470  // Add a keyword or replace a keyword with the value of another keyword.
471  // The keywords can be table or column keywords (col::key).
472  ValueHolder getRecFld (const String& name);
473 
474  // Define a field with the given data type in the Record.
475  static void setRecFld (RecordInterface& rec,
476  const String& name,
477  const String& dtype,
478  const ValueHolder& vh);
479 
480  // Get the type string. If empty, it is made from the given
481  // data type.
482  static String getTypeString (const String& typeStr, DataType type);
483 
484  // Add a keyword or replace a keyword with a value.
485  // The keyword can be a table or column keyword (col::key).
486  // The data type string can be empty leaving the data type unchanged.
487  void handleSetKey (const String& name, const String& dtype,
488  const ValueHolder& value);
489 
490  // Rename a table or column keyword.
491  void handleRenameKey (const String& oldName, const String& newName);
492 
493  // Remove a table or column keyword.
494  void handleRemoveKey (const String& name);
495 
496  // Split the given name into optional shorthand, column and fields.
497  // Find the keywordset for it and fill in the final keyword name.
498  // It is a helper function for handleSetKey, etc.
499  TableRecord& findKeyword (const String& name, String& keyName);
500 
501  // Add an update object.
502  void addUpdate (TableParseUpdate* upd);
503 
504  // Set the insert expressions for all rows.
505  void setInsertExprs (const std::vector<TableExprNode> exprs)
506  { insertExprs_p = exprs; }
507 
508  // Keep the update expressions.
509  void handleUpdate();
510 
511  // Make ready for the insert expression.
512  // The first one uses values (added via addUpdate),
513  // the second one a subquery.
514  // <group>
515  void handleInsert();
516  void handleInsert (TableParseSelect* sel);
517  // </group>
518 
519  // Make ready for a COUNT command.
520  // It checks if all column expressions are scalar.
521  void handleCount();
522 
523  // Keep the sort expressions.
524  void handleSort (const std::vector<TableParseSort>& sortList,
525  Bool noDuplicates, Sort::Order defaultSortOrder);
526 
527  // Evaluate and keep limit/offset/stride given as start:end:incr
528  void handleLimit (const TableExprNodeSetElem& expr);
529 
530  // Evaluate and keep the limit value.
531  void handleLimit (const TableExprNode& expr);
532 
533  // Evaluate and keep the offset value.
534  void handleOffset (const TableExprNode& expr);
535 
536  // Evaluate and add the rows.
537  void handleAddRow (const TableExprNode& expr);
538 
539  // Add a table nr, name, or object to the container.
540  void addTable (Int tabnr, const String& name,
541  const Table& table,
542  const String& shorthand,
543  Bool addToFromList,
544  const vector<const Table*> tempTables,
545  const vector<TableParseSelect*>& stack);
546 
547  // Make a Table object for given name, seqnr or so.
548  // If <src>alwaysOpen=False</src> the table will only be looked up,
549  // but not opened if not found. This is meant for concatenated tables
550  // in TaQLNodeHandler.
551  Table makeTable (Int tabnr, const String& name,
552  const Table& ftab,
553  const String& shorthand,
554  const vector<const Table*> tempTables,
555  const vector<TableParseSelect*>& stack,
556  Bool alwaysOpen=True);
557 
558  // Replace the first table (used by CALC command).
559  void replaceTable (const Table& table);
560 
561  // Find the keyword or column name and create a TableExprNode from it.
562  // If <src>tryProj=True</src> it is first tried if the column is a coluymn
563  // in the projected table (i.e., result from the SELECT part).
564  TableExprNode handleKeyCol (const String& name, Bool tryProj);
565 
566  // Handle a slice operator.
567  static TableExprNode handleSlice (const TableExprNode& array,
568  const TableExprNodeSet& indices,
569  const TaQLStyle&);
570 
571  // Handle a function.
572  TableExprNode handleFunc (const String& name,
573  const TableExprNodeSet& arguments,
574  const TaQLStyle&);
575 
576  // Make a function object node for the given function name and arguments.
577  // The ignoreFuncs vector contains invalid function codes.
578  static TableExprNode makeFuncNode (TableParseSelect*,
579  const String& name,
580  const TableExprNodeSet& arguments,
581  const Vector<int>& ignoreFuncs,
582  const Table& table,
583  const TaQLStyle&);
584 
585  // Add a column to the list of column names.
586  void handleColumn (Int type, const String& name, const TableExprNode& expr,
587  const String& newName, const String& nameMask,
588  const String& newDtype);
589 
590  // Finish the addition of columns to the list of column names.
591  void handleColumnFinish (Bool distinct);
592 
593  // Set the DataManager info for a new table.
594  void setDMInfo (const Record& dminfo)
595  { dminfo_p = dminfo;}
596 
597  // Handle the name and type given in a GIVING clause.
598  void handleGiving (const String& name, const Record& type);
599 
600  // Handle the set given in a GIVING clause.
601  void handleGiving (const TableExprNodeSet&);
602 
603  // Get the projected column names.
604  const Block<String>& getColumnNames() const;
605 
606  // Get the resulting table.
607  const Table& getTable() const;
608 
609  // An exception is thrown if the node uses an aggregate function.
610  static void checkAggrFuncs (const TableExprNode& node);
611 
612  // Show the structure of fromTables_p[0] using the options given in parts[2:].
613  String getTableInfo (const Vector<String>& parts, const TaQLStyle& style);
614 
615  // Split a name into its parts (shorthand, column and field names).
616  // True is returned if the name contained a keyword part.
617  // In that case fieldNames contains the keyword name and the possible
618  // subfields. The possible shorthand and the column name are
619  // filled in if it is a column keyword.
620  // If the name represents a column, fieldNames contains the subfields
621  // of the column (for the case where the column contains records).
622  // If the name is invalid, an exception is thrown if checkError=True.
623  // Otherwise the name is treated as a normal name without keyword.
624  // If allowEmtpy is True, :: is allowed, otherwise an error is thrown.
625  static Bool splitName (String& shorthand, String& columnName,
626  Vector<String>& fieldNames, const String& name,
627  Bool checkError, Bool isKeyword, Bool allowNoKey);
628 
629 private:
630  // Test if groupby or aggregate functions are given.
631  // <br> bit 0: on = groupby is given
632  // <br> bit 1: on = aggregate functions are given
633  // <br> bit 2: on = only select count(*) aggregate function is given
634  Int testGroupAggr (std::vector<TableExprNodeRep*>& aggr) const;
635 
636  // Get the aggregate functions used in SELECT and HAVING.
637  vector<TableExprNodeRep*> getAggrNodes() const;
638 
639  // Try to make a UDF function node for the given function name and arguments.
640  static TableExprNode makeUDFNode (TableParseSelect*,
641  const String& name,
642  const TableExprNodeSet& arguments,
643  const Table& table,
644  const TaQLStyle&);
645 
646  // Find the function code belonging to a function name.
647  // Functions to be ignored can be given (as function type values).
648  // If the function name is unknown, NRFUNC is returned.
649  static TableExprFuncNode::FunctionType findFunc
650  (const String& name,
651  uInt narguments,
652  const Vector<Int>& ignoreFuncs);
653 
654  // Do the update step.
655  // Rows 0,1,2,.. in UpdTable are updated from the expression result
656  // for the rows in the given rownrs vector.
657  void doUpdate (Bool showTimings, const Table& origTable,
658  Table& updTable, const Vector<uInt>& rownrs,
659  const CountedPtr<TableExprGroupResult>& groups =
661 
662  // Do the insert step and return a selection containing the new rows.
663  Table doInsert (Bool showTimings, Table& table);
664 
665  // Do the delete step.
666  void doDelete (Bool showTimings, Table& table);
667 
668  // Do the count step returning a memory table containing the unique
669  // column values and the counts of the column values.
670  Table doCount (Bool showTimings, const Table&);
671 
672  // Do the projection step returning a table containing the projection.
673  Table doProject (Bool showTimings, const Table&,
674  const CountedPtr<TableExprGroupResult>& groups =
676 
677  // Do the projection containing column expressions.
678  // Use the selected or unselected columns depending on <src>useSel</src>.
679  Table doProjectExpr (Bool useSel,
680  const CountedPtr<TableExprGroupResult>& groups);
681 
682  // Create a table using the given parameters.
683  // The variables set by handleGiven are used for name and type.
684  Table createTable (const TableDesc& td,
685  Int64 nrow, const Record& dmInfo);
686 
687  // Make the (empty) table for the epxression in the SELECT clause.
688  void makeProjectExprTable();
689 
690  // Fill projectExprSelColumn_p telling the columns to be projected
691  // at the first stage.
692  void makeProjectExprSel();
693 
694  // Add a column node to applySelNodes_p.
695  void addApplySelNode (const TableExprNode& node)
696  { applySelNodes_p.push_back (node); }
697 
698  // Set the selected rows for the column objects in applySelNodes_p.
699  // These nodes refer the original table. They requires different row
700  // numbers than the selected groups and projected columns.
701  // rownrs_p is changed to use row 0..n.
702  // It returns the Table containing the subset of rows in the input Table.
703  Table adjustApplySelNodes (const Table&);
704 
705  // Do the groupby/aggregate step and return its result.
707  (bool showTimings, const std::vector<TableExprNodeRep*> aggrNodes,
708  Int groupAggrUsed);
709 
710  // Do the HAVING step.
711  void doHaving (Bool showTimings,
712  const CountedPtr<TableExprGroupResult>& groups);
713 
714  // Do a groupby/aggregate step that only does a 'select count(*)'.
715  CountedPtr<TableExprGroupResult> doOnlyCountAll (TableExprNodeRep* aggrNode);
716 
717  // Do a full groupby/aggregate step.
719  (const std::vector<TableExprNodeRep*>& aggrNodes);
720 
721  // Do the sort step.
722  void doSort (Bool showTimings);
723 
724  // Do the limit/offset step.
725  void doLimOff (Bool showTimings);
726  Table doLimOff (Bool showTimings, const Table& table);
727 
728  // Do the 'select distinct' step.
729  Table doDistinct (Bool showTimings, const Table& table);
730 
731  // Finish the table (rename, copy, and/or flush).
732  Table doFinish (Bool showTimings, Table& table);
733 
734  // Update the values in the columns (helpers of doUpdate).
735  // <group>
736  template<typename TCOL, typename TNODE>
737  void updateValue (uInt row, const TableExprId& rowid,
738  Bool isScalarCol, const TableExprNode& node,
739  const Array<Bool>& mask, Bool maskFirst,
740  TableColumn& col, const Slicer* slicerPtr,
741  ArrayColumn<Bool>& maskCol);
742  template<typename TCOL, typename TNODE>
743  void updateScalar (uInt row, const TableExprId& rowid,
744  const TableExprNode& node,
745  TableColumn& col);
746  template<typename TCOL, typename TNODE>
747  void updateArray (uInt row, const TableExprId& rowid,
748  const TableExprNode& node,
749  const Array<TNODE>& res,
750  ArrayColumn<TCOL>& col);
751  template<typename TCOL, typename TNODE>
752  void updateSlice (uInt row, const TableExprId& rowid,
753  const TableExprNode& node,
754  const Array<TNODE>& res,
755  const Slicer& slice,
756  ArrayColumn<TCOL>& col);
757  template<typename TCOL, typename TNODE>
758  void copyMaskedValue (uInt row, ArrayColumn<TCOL>& acol,
759  const Slicer* slicerPtr,
760  const TNODE* val,
761  uInt incr, const Array<Bool>& mask);
762  Array<Bool> makeMaskSlice (const Array<Bool>& mask,
763  Bool maskFirst,
764  const IPosition& shapeCol,
765  const Slicer* slicerPtr);
766  void checkMaskColumn (Bool hasMask,
767  const ArrayColumn<Bool>& maskCol,
768  const TableColumn& col);
769  // </group>
770 
771  // Make a data type from the string.
772  // It checks if it is compatible with the given (expression) data type.
773  DataType makeDataType (DataType dtype, const String& dtstr,
774  const String& colName);
775 
776  // Get the order for this key. Use the default order_p if not
777  // explicitly given with the key.
778  Sort::Order getOrder (const TableParseSort& key) const;
779 
780  // Make an array from the contents of a column in a subquery.
781  TableExprNode getColSet();
782 
783  // Make a set from the results of the subquery.
784  TableExprNode makeSubSet() const;
785 
786  // Evaluate an int scalar expression.
787  Int64 evalIntScaExpr (const TableExprNode& expr) const;
788 
789  // Find a table for the given shorthand.
790  // Optionally the WITH tables are searched as well.
791  // If no shorthand is given, the first table is returned (if there).
792  // If not found, a null Table object is returned.
793  Table findTable (const String& shorthand, Bool doWith) const;
794 
795  // Handle the selection of a wildcarded column name.
796  void handleWildColumn (Int stringType, const String& name);
797 
798  // Add the description of a column to the table description.
799  // ndim < 0 means a scalar column.
800  void addColumnDesc (TableDesc& td, DataType dtype,
801  const String& colName, Int options,
802  Int ndim, const IPosition& shape,
803  const String& dmType, const String& dmGroup,
804  const String& comment,
805  const TableRecord& keywordSet,
806  const Vector<String>& unitName,
807  const Record& attributes);
808 
809  // Find the names of all stored columns in a table.
810  Block<String> getStoredColumns (const Table& tab) const;
811 
812  // Try to find the keyword representing a table in one of the tables
813  // in any select block (from inner to outer).
814  // If not found, an exception is thrown.
815  static Table tableKey (const String& fullName,
816  const String& shorthand, const String& columnName,
817  const Vector<String>& fieldNames,
818  const vector<TableParseSelect*>& stack);
819 
820  // Try to find the keyword representing a table in the given table.
821  // If the columnName is empty, the keyword is a table keyword.
822  // If not found, a null Table object is returned.
823  static Table findTableKey (const Table& table, const String& columnName,
824  const Vector<String>& keyNames);
825 
826  // Check if the tables used in selection columns have the same
827  // size as the first table given in FROM.
828  void checkTableProjSizes() const;
829 
830  // Create the set of aggregate functions and groupby keys in case
831  // a single groupby key is given.
832  // This offers much faster map access then doGroupByAggrMultiple.
833  template<typename T>
834  std::vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrSingleKey
835  (const vector<TableExprNodeRep*>& aggrNodes)
836  {
837  // We have to group the data according to the (possibly empty) groupby.
838  // We step through the table in the normal order which may not be the
839  // groupby order.
840  // A map<key,int> is used to keep track of the results where the int
841  // is the index in a vector of a set of aggregate function objects.
842  vector<CountedPtr<TableExprGroupFuncSet> > funcSets;
843  std::map<T, int> keyFuncMap;
844  T lastKey = std::numeric_limits<T>::max();
845  int groupnr = -1;
846  // Loop through all rows.
847  // For each row generate the key to get the right entry.
848  TableExprId rowid(0);
849  T key;
850  for (uInt i=0; i<rownrs_p.size(); ++i) {
851  rowid.setRownr (rownrs_p[i]);
852  groupbyNodes_p[0].get (rowid, key);
853  if (key != lastKey) {
854  typename std::map<T, int>::iterator iter = keyFuncMap.find (key);
855  if (iter == keyFuncMap.end()) {
856  groupnr = funcSets.size();
857  keyFuncMap[key] = groupnr;
858  funcSets.push_back (new TableExprGroupFuncSet (aggrNodes));
859  } else {
860  groupnr = iter->second;
861  }
862  }
863  rowid.setRownr (rownrs_p[i]);
864  funcSets[groupnr]->apply (rowid);
865  }
866  return funcSets;
867  }
868 
869  // Create the set of aggregate functions and groupby keys in case
870  // multiple keys are given.
871  std::vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrMultipleKeys
872  (const vector<TableExprNodeRep*>& aggrNodes);
873 
874  //# Command type.
876  //# Table description for a series of column descriptions.
878  //# Vector of TableParse objects (from WITH and FROM clause).
879  //# This is needed for the functions above, otherwise they have no
880  //# way to communicate.
881  vector<TableParse> withTables_p;
882  vector<TableParse> fromTables_p;
883  //# Block of selected column names (new name in case of select).
885  //# Block of selected mask column names (for masked arrays).
887  //# Block of selected column expressions.
889  //# The old name for a selected column.
891  //# The new data type for a column.
893  //# The keywords used in a column.
895  //# Number of real expressions used in selected columns.
897  //# Distinct values in output?
899  //# Name and type of the resulting table (from GIVING part).
901  uInt resultType_p; //# 0-unknown 1=memory 2=scratch 3=plain
902  Bool resultCreated_p; //# Has the result table been created?
907  //# Resulting set (from GIVING part).
909  //# The WHERE expression tree.
911  //# The GROUPBY expressions.
912  vector<TableExprNode> groupbyNodes_p;
913  Bool groupbyRollup_p; //# use ROLLUP in GROUPBY?
914  //# The HAVING expression.
916  //# The possible limit (= max nr of selected rows) (0 means no limit).
918  //# The possible last row (0 means no end; can be <0).
919  //# limit_p and endrow_p cannot be both !=0.
921  //# The possible offset (= nr of selected rows to skip).
923  //# The possible stride in offset:endrow:stride.
925  //# The update and insert list.
926  std::vector<TableParseUpdate*> update_p;
927  //# The insert expressions (possibly for multiple rows).
928  std::vector<TableExprNode> insertExprs_p;
929  //# The table selection to be inserted.
931  //# The sort list.
932  std::vector<TableParseSort> sort_p;
933  //# The noDuplicates sort switch.
935  //# The default sort order.
937  //# All nodes that need to be adjusted for a selection of rownrs.
938  //# It can consist of column nodes and the rowid function node.
939  //# Some nodes (in aggregate functions) can later be disabled for adjustment.
940  vector<TableExprNode> applySelNodes_p;
941  //# The resulting table.
943  //# The first table used when creating a column object.
944  //# All other tables used for them should have the same size.
947  //# The table resulting from a projection with expressions.
949  //# The projected columns used in the HAVING and ORDERBY clauses.
952  //# The resulting row numbers.
954 };
955 
956 
957 
958 //# Implement the inline functions.
959 inline Bool TableParse::test (const String& str) const
960  { return (shorthand_p == str ? True : False); }
961 
962 inline const String& TableParse::shorthand() const
963  { return shorthand_p; }
964 
965 inline const Table& TableParse::table() const
966  { return table_p; }
967 
968 
969 inline void TableParseUpdate::setColumnName (const String& name)
970  { columnName_p = name; }
972  { columnNameMask_p = name; }
974  { return columnName_p; }
976  { return columnNameMask_p; }
978  { return indexPtr_p; }
980  { return indexNode_p; }
982  { return node_p; }
984  { return node_p; }
985 inline void TableParseUpdate::adaptUnit (const Unit& columnUnit)
986  { node_p.adaptUnit (columnUnit); }
987 
988 inline const TableExprNode& TableParseSort::node() const
989  { return node_p; }
991  { return given_p; }
993  { return order_p; }
994 
995 
997  { return columnNames_p; }
998 
999 inline const Table& TableParseSelect::getTable() const
1000  { return table_p; }
1001 
1003  { update_p.push_back (upd); }
1004 
1006  { return (key.orderGiven() ? key.order() : order_p); }
1007 
1008 
1009 } //# NAMESPACE CASACORE - END
1010 
1011 #endif
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:119
Block< Bool > projectExprSelColumn_p
Definition: TableParse.h:951
A 1-D Specialization of the Array class.
Definition: ArrayIO.h:45
vector< TableExprNode > groupbyNodes_p
Definition: TableParse.h:912
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
void setColumnName(const String &name)
Set the column name.
Definition: TableParse.h:969
int Int
Definition: aipstype.h:50
TableExprNodeSet * resultSet_p
Definition: TableParse.h:908
Bool orderGiven() const
Is the order given?
Definition: TableParse.h:990
EndianFormat
Define the possible endian formats in which table data can be stored.
Definition: Table.h:192
Main interface class to a read/write table.
Definition: Table.h:153
TableExprNodeIndex * indexPtr_p
Definition: TableParse.h:339
LatticeExprNode mask(const LatticeExprNode &expr)
This function returns the mask of the given expression.
std::vector< TableExprNode > insertExprs_p
Definition: TableParse.h:928
vector< TableParse > withTables_p
Definition: TableParse.h:881
TableExprNode indexNode_p
Definition: TableParse.h:340
TableExprNode array(const TableExprNode &values, const TableExprNodeSet &shape)
Create an array of the given shape and fill it with the values.
Definition: ExprNode.h:2105
Class to hold multiple table expression nodes.
Definition: ExprNodeSet.h:311
LatticeExprNode max(const LatticeExprNode &left, const LatticeExprNode &right)
Handle class for a table column expression tree.
Definition: ExprNode.h:616
TableExprNode node_p
Definition: TableParse.h:229
Order
Enumerate the sort order:
Definition: Sort.h:260
void setRownr(uInt rownr)
Set the row number.
Definition: TableExprId.h:185
void setColumnNameMask(const String &name)
Set the column name forthe mask.
Definition: TableParse.h:971
TableExprNodeIndex * indexPtr() const
Get the pointer to the indices.
Definition: TableParse.h:977
Sort::Order getOrder(const TableParseSort &key) const
Get the order for this key.
Definition: TableParse.h:1005
void adaptUnit(const Unit &columnUnit)
Adapt the possible unit of the expression to the possible unit of the column.
Definition: TableParse.h:985
Abstract base class for a node in a table column expression tree.
Definition: ExprNodeRep.h:157
TableParse & operator=(const TableParse &)
Assignment (copy semantics).
Block< String > columnNameMasks_p
Definition: TableParse.h:886
const Table & table() const
Get table object.
Definition: TableParse.h:965
Options defining how table files are organized.
Definition: StorageOption.h:71
defines physical units
Definition: Unit.h:189
vector< TableExprNode > applySelNodes_p
Definition: TableParse.h:940
vector< TableParse > fromTables_p
Definition: TableParse.h:882
Select-class for flex/bison scanner/parser for TableParse.
Definition: TableParse.h:381
Sort::Order order() const
Get the sort order.
Definition: TableParse.h:992
Referenced counted pointer for constant data.
Definition: CountedPtr.h:80
Bool maskFirst() const
Tell if the mask is given first (i.e., before slice).
Definition: TableParse.h:312
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:981
Table::EndianFormat endianFormat_p
Definition: TableParse.h:904
Class to hold the table expression nodes for an element in a set.
Definition: ExprNodeSet.h:94
Class to hold the result of a TaQL command.
Definition: TaQLResult.h:67
Vector< uInt > rownrs_p
Definition: TableParse.h:953
A holder for a value of any basic Casacore data type.
Definition: ValueHolder.h:67
Class with static members defining the TaQL style.
Definition: TaQLStyle.h:64
LatticeExprNode ndim(const LatticeExprNode &expr)
1-argument function to get the dimensionality of a lattice.
CommandType commandType() const
Return the command type.
Definition: TableParse.h:409
Block< uInt > projectExprSubset_p
Definition: TableParse.h:950
Block< String > columnDtypes_p
Definition: TableParse.h:892
const TableExprNode & mask() const
Get the mask.
Definition: TableParse.h:328
const Block< String > & getColumnNames() const
Get the projected column names.
Definition: TableParse.h:996
const String & columnNameMask() const
Get the possible column name for the mask.
Definition: TableParse.h:975
Block< TableRecord > columnKeywords_p
Definition: TableParse.h:894
A hierarchical collection of named fields of various types.
Definition: Record.h:180
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
Bool test(const String &shortHand) const
Test if shorthand matches.
Definition: TableParse.h:959
Read/write access to a table column.
Definition: TableColumn.h:98
Class containing the results of aggregated values in a group.
Definition: ExprGroup.h:801
Block< String > columnNames_p
Definition: TableParse.h:884
const Bool False
Definition: aipstype.h:44
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape...
Definition: ExprNode.h:2163
A hierarchical collection of named fields of various types.
Definition: TableRecord.h:182
Block< TableExprNode > columnExpr_p
Definition: TableParse.h:888
TableExprNode getNode() const
Return the expression node.
Definition: TableParse.h:413
TableParse()
Default constructor for container class.
Helper class for sort keys in TableParse.
Definition: TableParse.h:199
Specify which elements to extract from an n-dimensional array.
Definition: Slicer.h:289
simple 1-D array
Definition: ArrayIO.h:47
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:988
Class to hold values from table grammar parser.
Definition: TableParse.h:109
TableExprNode havingNode_p
Definition: TableParse.h:915
TableParseSelect * insSel_p
Definition: TableParse.h:930
The identification of a TaQL selection subject.
Definition: TableExprId.h:97
void addUpdate(TableParseUpdate *upd)
Add an update object.
Definition: TableParse.h:1002
void setInsertExprs(const std::vector< TableExprNode > exprs)
Set the insert expressions for all rows.
Definition: TableParse.h:505
Block< String > columnOldNames_p
Definition: TableParse.h:890
const TableExprNode & indexNode() const
Get the index expression node.
Definition: TableParse.h:979
String: the storage and methods of handling collections of characters.
Definition: String.h:223
Define the structure of a Casacore table.
Definition: TableDesc.h:186
const Table & getTable() const
Get the resulting table.
Definition: TableParse.h:999
TaQLResult tableCommand(const String &command)
Abstract base class for Record classes.
StorageOption storageOption_p
Definition: TableParse.h:903
const String & shorthand() const
Get the shorthand.
Definition: TableParse.h:962
The index of an array element in a table select expression.
void addApplySelNode(const TableExprNode &node)
Add a column node to applySelNodes_p.
Definition: TableParse.h:695
const Bool True
Definition: aipstype.h:43
const String & columnName() const
Get the column name.
Definition: TableParse.h:973
this file contains all the compiler specific defines
Definition: mainpage.dox:28
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
unsigned int uInt
Definition: aipstype.h:51
Helper class for updates in TableParse.
Definition: TableParse.h:263
std::vector< TableParseUpdate * > update_p
Definition: TableParse.h:926
std::vector< TableParseSort > sort_p
Definition: TableParse.h:932
void setDMInfo(const Record &dminfo)
Set the DataManager info for a new table.
Definition: TableParse.h:594