Skip to content

M
mariadb
Commit 8ba013a2 authored by Olivier Bertrand's avatar Olivier Bertrand
Browse files
Options

- Serialize: Protect again eventual longjmp's. 

Always return NULL on error.
Adding also the file length.
  modified:   storage/connect/json.cpp
  modified:   storage/connect/jsonudf.cpp

- JSONCOL::WriteColumn Add types SHORT and BIGINT as accepted
  modified:   storage/connect/tabjson.cpp

- TDBJSN: Make this type use a separate storage for Json parsing
and retrieve this memory between each rows. This is necessary
to be able to handle big tables. See MDEV-9228.
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h
parent d059dd73
Hide whitespace changes
Inline Side-by-side
Showing with 140 additions and 84 deletions
storage/connect/json.cpp
View file @ 8ba013a2
......@@ -534,15 +534,27 @@ PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src)
/***********************************************************************/
PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty)
{
PSZ str = NULL;
bool b = false, err = true;
JOUT *jp;
FILE *fs = NULL;
g->Message[0] = 0;
if (!jsp) {
// Save stack and allocation environment and prepare error return
if (g->jump_level == MAX_JUMP) {
strcpy(g->Message, MSG(TOO_MANY_JUMPS));
return NULL;
} // endif jump_level
if (setjmp(g->jumper[++g->jump_level])) {
str = NULL;
goto fin;
} // endif jmp
if (!jsp) {
strcpy(g->Message, "Null json tree");
return NULL;
goto fin;
} else if (!fn) {
// Serialize to a string
jp = new(g) JOUTSTR(g);
......@@ -552,7 +564,7 @@ PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty)
sprintf(g->Message, MSG(OPEN_MODE_ERROR),
"w", (int)errno, fn);
strcat(strcat(g->Message, ": "), strerror(errno));
return g->Message;
goto fin;;
} else if (pretty >= 2) {
// Serialize to a pretty file
jp = new(g)JOUTPRT(g, fs);
......@@ -582,20 +594,20 @@ PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty)
if (fs) {
fputs(EL, fs);
fclose(fs);
return (err) ? g->Message : NULL;
str = (err) ? NULL : "Ok";
} else if (!err) {
PSZ str = ((JOUTSTR*)jp)->Strp;
str = ((JOUTSTR*)jp)->Strp;
jp->WriteChr('\0');
PlugSubAlloc(g, NULL, ((JOUTSTR*)jp)->N);
return str;
} else {
if (!g->Message[0])
strcpy(g->Message, "Error in Serialize");
return NULL;
} // endif's
fin:
g->jump_level--;
return str;
} // end of Serialize
/***********************************************************************/
......
storage/connect/jsonudf.cpp
View file @ 8ba013a2
......@@ -1133,8 +1133,7 @@ static char *MakeResult(PGLOBAL g, UDF_ARGS *args, PJSON top, uint n = 2)
if (IsJson(args, 0) == 2) {
// Make the change in the json file
char *msg;
int pretty = 2;
int pretty = 2;
for (uint i = n; i < args->arg_count; i++)
if (args->arg_type[i] == INT_RESULT) {
......@@ -1142,8 +1141,8 @@ static char *MakeResult(PGLOBAL g, UDF_ARGS *args, PJSON top, uint n = 2)
break;
} // endif type
if ((msg = Serialize(g, top, MakePSZ(g, args, 0), pretty)))
PUSH_WARNING(msg);
if (!Serialize(g, top, MakePSZ(g, args, 0), pretty))
PUSH_WARNING(g->Message);
str = NULL;
} else if (IsJson(args, 0) == 3) {
......@@ -1151,10 +1150,8 @@ static char *MakeResult(PGLOBAL g, UDF_ARGS *args, PJSON top, uint n = 2)
if (bsp->Filename) {
// Make the change in the json file
char *msg;
if ((msg = Serialize(g, top, bsp->Filename, bsp->Pretty)))
PUSH_WARNING(msg);
if (!Serialize(g, top, bsp->Filename, bsp->Pretty))
PUSH_WARNING(g->Message);
str = bsp->Filename;
} else if (!(str = Serialize(g, top, NULL, 0)))
......@@ -3810,7 +3807,7 @@ my_bool jfile_make_init(UDF_INIT *initid, UDF_ARGS *args, char *message)
char *jfile_make(UDF_INIT *initid, UDF_ARGS *args, char *result,
unsigned long *res_length, char *is_null, char *)
{
char *p, *msg, *str = NULL, *fn = NULL;
char *p, *str = NULL, *fn = NULL;
int n, pretty = 2;
PJSON jsp;
PJVAL jvp;
......@@ -3878,9 +3875,9 @@ char *jfile_make(UDF_INIT *initid, UDF_ARGS *args, char *result,
} // endswitch arg_type
if (fn) {
if ((msg = Serialize(g, jvp->GetJson(), fn, pretty)))
PUSH_WARNING(msg);
} else
if (!Serialize(g, jvp->GetJson(), fn, pretty))
PUSH_WARNING(g->Message);
} else
PUSH_WARNING("Missing file name");
str= fn;
......
storage/connect/tabjson.cpp
View file @ 8ba013a2
......@@ -38,6 +38,7 @@
/***********************************************************************/
#define MAXCOL 200 /* Default max column nb in result */
#define TYPE_UNKNOWN 12 /* Must be greater than other types */
#define USE_G 1 /* Use recoverable memory if 1 */
/***********************************************************************/
/* External function. */
......@@ -411,9 +412,23 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m)
// Txfp must be set for TDBDOS
tdbp = new(g) TDBJSN(this, txfp);
} else {
#if USE_G
// Allocate the parse work memory
PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL));
memset(G, 0, sizeof(GLOBAL));
G->Sarea_Size = Lrecl * 10;
G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size);
PlugSubSet(G, G->Sarea, G->Sarea_Size);
G->jump_level = -1;
((TDBJSN*)tdbp)->G = G;
#else
((TDBJSN*)tdbp)->G = g;
#endif
} else {
txfp = new(g) MAPFAM(this);
tdbp = new(g) TDBJSON(this, txfp);
((TDBJSON*)tdbp)->G = g;
} // endif Pretty
if (Multiple)
......@@ -462,6 +477,7 @@ TDBJSN::TDBJSN(PJDEF tdp, PTXF txfp) : TDBDOS(tdp, txfp)
TDBJSN::TDBJSN(TDBJSN *tdbp) : TDBDOS(NULL, tdbp)
{
G = NULL;
Top = tdbp->Top;
Row = tdbp->Row;
Val = tdbp->Val;
......@@ -485,6 +501,7 @@ TDBJSN::TDBJSN(TDBJSN *tdbp) : TDBDOS(NULL, tdbp)
// Used for update
PTDB TDBJSN::CopyOne(PTABS t)
{
G = NULL;
PTDB tp;
PJCOL cp1, cp2;
PGLOBAL g = t->G;
......@@ -603,7 +620,7 @@ bool TDBJSN::OpenDB(PGLOBAL g)
return true;
} // endswitch Jmode
} // endif Use
} // endif Use
return TDBDOS::OpenDB(g);
} // end of OpenDB
......@@ -655,19 +672,26 @@ int TDBJSN::ReadDB(PGLOBAL g)
NextSame = 0;
M++;
return RC_OK;
} else if ((rc = TDBDOS::ReadDB(g)) == RC_OK)
if (!IsRead() && ((rc = ReadBuffer(g)) != RC_OK)) {
// Deferred reading failed
} else if (!(Row = ParseJson(g, To_Line,
strlen(To_Line), &Pretty, &Comma))) {
rc = (Pretty == 1 && !strcmp(To_Line, "]")) ? RC_EF : RC_FX;
} else {
Row = FindRow(g);
SameRow = 0;
Fpos++;
M = 1;
rc = RC_OK;
} // endif's
} else if ((rc = TDBDOS::ReadDB(g)) == RC_OK) {
if (!IsRead() && ((rc = ReadBuffer(g)) != RC_OK))
// Deferred reading failed
return rc;
#if USE_G
// Recover the memory used for parsing
PlugSubSet(G, G->Sarea, G->Sarea_Size);
#endif
if ((Row = ParseJson(G, To_Line, strlen(To_Line), &Pretty, &Comma))) {
Row = FindRow(g);
SameRow = 0;
Fpos++;
M = 1;
rc = RC_OK;
} else
rc = (Pretty == 1 && !strcmp(To_Line, "]")) ? RC_EF : RC_FX;
} // endif ReadDB
return rc;
} // end of ReadDB
......@@ -744,7 +768,7 @@ int TDBJSN::MakeTopTree(PGLOBAL g, PJSON jsp)
if (MakeTopTree(g, Row))
return true;
if ((s = Serialize(g, Top, NULL, Pretty))) {
if ((s = Serialize(G, Top, NULL, Pretty))) {
if (Comma)
strcat(s, ",");
......@@ -761,15 +785,30 @@ int TDBJSN::MakeTopTree(PGLOBAL g, PJSON jsp)
} // end of PrepareWriting
/* ---------------------------- JSONCOL ------------------------------ */
/***********************************************************************/
/* WriteDB: Data Base write routine for DOS access method. */
/***********************************************************************/
int TDBJSN::WriteDB(PGLOBAL g)
{
int rc = TDBDOS::WriteDB(g);
#if USE_G
PlugSubSet(G, G->Sarea, G->Sarea_Size);
#endif
Row->Clear();
return rc;
} // end of WriteDB
/* ---------------------------- JSONCOL ------------------------------ */
/***********************************************************************/
/* JSONCOL public constructor. */
/***********************************************************************/
JSONCOL::JSONCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i)
: DOSCOL(g, cdp, tdbp, cprec, i, "DOS")
{
{
Tjp = (TDBJSN *)(tdbp->GetOrig() ? tdbp->GetOrig() : tdbp);
G = Tjp->G;
Jpath = cdp->GetFmt();
MulVal = NULL;
Nodes = NULL;
......@@ -777,14 +816,15 @@ JSONCOL::JSONCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i)
Xnod = -1;
Xpd = false;
Parsed = false;
} // end of JSONCOL constructor
} // end of JSONCOL constructor
/***********************************************************************/
/* JSONCOL constructor used for copying columns. */
/* tdbp is the pointer to the new table descriptor. */
/***********************************************************************/
JSONCOL::JSONCOL(JSONCOL *col1, PTDB tdbp) : DOSCOL(col1, tdbp)
{
{
G = col1->G;
Tjp = col1->Tjp;
Jpath = col1->Jpath;
MulVal = col1->MulVal;
......@@ -793,7 +833,7 @@ JSONCOL::JSONCOL(JSONCOL *col1, PTDB tdbp) : DOSCOL(col1, tdbp)
Xnod = col1->Xnod;
Xpd = col1->Xpd;
Parsed = col1->Parsed;
} // end of JSONCOL copy constructor
} // end of JSONCOL copy constructor
/***********************************************************************/
/* SetBuffer: prepare a column block for write operation. */
......@@ -808,6 +848,7 @@ bool JSONCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check)
return true;
Tjp = (TDBJSN*)To_Tdb;
G = Tjp->G;
return false;
} // end of SetBuffer
......@@ -1103,7 +1144,7 @@ PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i)
Value->SetValue(row->GetType() == TYPE_JAR ? row->size() : 1);
return(Value);
} else if (Nodes[i].Op == OP_XX) {
return MakeJson(g, row);
return MakeJson(G, row);
} else switch (row->GetType()) {
case TYPE_JOB:
if (!Nodes[i].Key) {
......@@ -1111,7 +1152,7 @@ PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i)
if (i < Nod-1)
continue;
else
val = new(g) JVALUE(row);
val = new(G) JVALUE(row);
} else
val = ((PJOB)row)->GetValue(Nodes[i].Key);
......@@ -1319,15 +1360,15 @@ PJSON JSONCOL::GetRow(PGLOBAL g)
break;
else if (!Nodes[i].Key)
// Construct intermediate array
nwr = new(g) JARRAY;
nwr = new(G) JARRAY;
else
nwr = new(g) JOBJECT;
nwr = new(G) JOBJECT;
if (row->GetType() == TYPE_JOB) {
((PJOB)row)->SetValue(g, new(g) JVALUE(nwr), Nodes[i-1].Key);
((PJOB)row)->SetValue(G, new(G) JVALUE(nwr), Nodes[i-1].Key);
} else if (row->GetType() == TYPE_JAR) {
((PJAR)row)->AddValue(g, new(g) JVALUE(nwr));
((PJAR)row)->InitArray(g);
((PJAR)row)->AddValue(G, new(G) JVALUE(nwr));
((PJAR)row)->InitArray(G);
} else {
strcpy(g->Message, "Wrong type when writing new row");
nwr = NULL;
......@@ -1384,21 +1425,21 @@ void JSONCOL::WriteColumn(PGLOBAL g)
if (Nodes[Nod-1].Op == OP_XX) {
s = Value->GetCharValue();
if (!(jsp = ParseJson(g, s, (int)strlen(s)))) {
if (!(jsp = ParseJson(G, s, (int)strlen(s)))) {
strcpy(g->Message, s);
longjmp(g->jumper[g->jump_level], 666);
} // endif jsp
if (arp) {
if (Nod > 1 && Nodes[Nod-2].Op == OP_EQ)
arp->SetValue(g, new(g) JVALUE(jsp), Nodes[Nod-2].Rank);
arp->SetValue(G, new(G) JVALUE(jsp), Nodes[Nod-2].Rank);
else
arp->AddValue(g, new(g) JVALUE(jsp));
arp->AddValue(G, new(G) JVALUE(jsp));
arp->InitArray(g);
arp->InitArray(G);
} else if (objp) {
if (Nod > 1 && Nodes[Nod-2].Key)
objp->SetValue(g, new(g) JVALUE(jsp), Nodes[Nod-2].Key);
objp->SetValue(G, new(G) JVALUE(jsp), Nodes[Nod-2].Key);
} else if (jvp)
jvp->SetValue(jsp);
......@@ -1409,17 +1450,19 @@ void JSONCOL::WriteColumn(PGLOBAL g)
// Passthru
case TYPE_DATE:
case TYPE_INT:
case TYPE_DOUBLE:
case TYPE_SHORT:
case TYPE_BIGINT:
case TYPE_DOUBLE:
if (arp) {
if (Nodes[Nod-1].Op == OP_EQ)
arp->SetValue(g, new(g) JVALUE(g, Value), Nodes[Nod-1].Rank);
arp->SetValue(G, new(G) JVALUE(G, Value), Nodes[Nod-1].Rank);
else
arp->AddValue(g, new(g) JVALUE(g, Value));
arp->AddValue(G, new(G) JVALUE(G, Value));
arp->InitArray(g);
arp->InitArray(G);
} else if (objp) {
if (Nodes[Nod-1].Key)
objp->SetValue(g, new(g) JVALUE(g, Value), Nodes[Nod-1].Key);
objp->SetValue(G, new(G) JVALUE(G, Value), Nodes[Nod-1].Key);
} else if (jvp)
jvp->SetValue(Value);
......@@ -1835,8 +1878,7 @@ void TDBJSON::CloseDB(PGLOBAL g)
return;
// Save the modified document
char filename[_MAX_PATH];
PSZ msg;
char filename[_MAX_PATH];
Doc->InitArray(g);
......@@ -1844,8 +1886,8 @@ void TDBJSON::CloseDB(PGLOBAL g)
PlugSetPath(filename, ((PJDEF)To_Def)->Fn, GetPath());
// Serialize the modified table
if ((msg = Serialize(g, Top, filename, Pretty)))
puts(msg);
if (!Serialize(g, Top, filename, Pretty))
puts(g->Message);
} // end of CloseDB
......
storage/connect/tabjson.h
View file @ 8ba013a2
......@@ -68,7 +68,8 @@ class JSONDEF : public DOSDEF { /* Table description */
/***********************************************************************/
class TDBJSN : public TDBDOS {
friend class JSONCOL;
public:
friend class JSONDEF;
public:
// Constructor
TDBJSN(PJDEF tdp, PTXF txfp);
TDBJSN(TDBJSN *tdbp);
......@@ -90,32 +91,34 @@ class TDBJSN : public TDBDOS {
virtual int Cardinality(PGLOBAL g);
virtual int GetMaxSize(PGLOBAL g);
virtual bool OpenDB(PGLOBAL g);
virtual bool PrepareWriting(PGLOBAL g);
virtual int ReadDB(PGLOBAL g);
virtual bool PrepareWriting(PGLOBAL g);
virtual int WriteDB(PGLOBAL g);
protected:
PJSON FindRow(PGLOBAL g);
int MakeTopTree(PGLOBAL g, PJSON jsp);
// Members
PJSON Top; // The top JSON tree
PJSON Row; // The current row
PJSON Val; // The value of the current row
PJCOL Colp; // The multiple column
JMODE Jmode; // MODE_OBJECT by default
char *Objname; // The table object name
char *Xcol; // Name of expandable column
int Fpos; // The current row index
int N; // The current Rownum
int M; // Index of multiple value
int Limit; // Limit of multiple values
int Pretty; // Depends on file structure
int NextSame; // Same next row
int SameRow; // Same row nb
int Xval; // Index of expandable array
int B; // Array index base
bool Strict; // Strict syntax checking
bool Comma; // Row has final comma
PGLOBAL G; // Support of parse memory
PJSON Top; // The top JSON tree
PJSON Row; // The current row
PJSON Val; // The value of the current row
PJCOL Colp; // The multiple column
JMODE Jmode; // MODE_OBJECT by default
char *Objname; // The table object name
char *Xcol; // Name of expandable column
int Fpos; // The current row index
int N; // The current Rownum
int M; // Index of multiple value
int Limit; // Limit of multiple values
int Pretty; // Depends on file structure
int NextSame; // Same next row
int SameRow; // Same row nb
int Xval; // Index of expandable array
int B; // Array index base
bool Strict; // Strict syntax checking
bool Comma; // Row has final comma
}; // end of class TDBJSN
/* -------------------------- JSONCOL class -------------------------- */
......@@ -154,7 +157,8 @@ class JSONCOL : public DOSCOL {
JSONCOL(void) {}
// Members
TDBJSN *Tjp; // To the JSN table block
PGLOBAL G; // Support of parse memory
TDBJSN *Tjp; // To the JSN table block
PVAL MulVal; // To value used by multiple column
char *Jpath; // The json path
JNODE *Nodes; // The intermediate objects
......@@ -170,7 +174,8 @@ class JSONCOL : public DOSCOL {
/* This is the JSON Access Method class declaration. */
/***********************************************************************/
class TDBJSON : public TDBJSN {
friend class JSONCOL;
friend class JSONDEF;
friend class JSONCOL;
public:
// Constructor
TDBJSON(PJDEF tdp, PTXF txfp);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7