Zsh Mailing List Archive
Messages sorted by: Reverse Date, Date, Thread, Author

Re: [PATCH] db/gdbm rewrite



On Thu, Feb 16, 2017, at 04:52 AM, Peter Stephenson wrote:
> However, replacing all the zsfree()s with free gave me an infinite loop
> on a free.  This was at free(umkey) inside the braces (the first
> of the two) in gdbmgetfn().  So the internal calculation of what needs
> freeing is definitely getting confused by something that's going on.

This leads to something. The zsfree() backtrace might be late effect of
previous incorrect zsfree(). I did following change:

-            /* Unmetafy */
-            char *umval = ztrdup(val);
-            umval = unmetafy(umval,&umlen);
+            /* Unmetafy with exact zalloc size */
+            char *umval = unmetafy_zalloc(val,&umlen);

in multiple unmetafy-places. Otherwise strlen() is shorter than actual
buffer size. Sending as incremental patch including your warnings catch,
also as complete patch, and as db_gdbm.c file.



The added function:
/*
 * Unmetafy that:
 * - duplicates bufer to work on it,
 * - does zalloc of exact size for the new string,
 * - restores work buffer to original content, to restore strlen
 *
 * No zsfree()-confusing string will be produced.
 */
char *unmetafy_zalloc(const char *to_copy, int *new_len) {
    char *work, *to_return;
    int my_new_len = 0;

    work = ztrdup(to_copy);
    work = unmetafy(work,&my_new_len);

    if (new_len)
        *new_len = my_new_len;

    /* This string can be correctly zsfree()-d */
    to_return = (char *) zalloc((my_new_len+1)*sizeof(char));
    memcpy(to_return, work, sizeof(char)*my_new_len); // memcpy handles
    $'\0'
    to_return[my_new_len]='\0';

    /* Restore original strlen and correctly free */
    strcpy(work, to_copy);
    zsfree(work);

    return to_return;
}

-- 
  Sebastian Gniazdowski
  psprint2@xxxxxxxxxxxx
diff --git a/Src/Modules/db_gdbm.c b/Src/Modules/db_gdbm.c
index 3eef4c6..0a16540 100644
--- a/Src/Modules/db_gdbm.c
+++ b/Src/Modules/db_gdbm.c
@@ -41,6 +41,7 @@
 static Param createhash( char *name, int flags );
 static int append_tied_name( const char *name );
 static int remove_tied_name( const char *name );
+char *unmetafy_zalloc(const char *to_copy, int *new_len);
 
 /*
  * Make sure we have all the bits I'm using for memory mapping, otherwise
@@ -228,7 +229,7 @@ bin_zuntie(char *nam, char **args, Options ops, UNUSED(int func))
 static int
 bin_zgdbmpath(char *nam, char **args, Options ops, UNUSED(int func))
 {
-    Param pm, reply_pm;
+    Param pm;
     char *pmname;
 
     pmname = *args;
@@ -294,9 +295,8 @@ gdbmgetfn(Param pm)
 
     /* Unmetafy key. GDBM fits nice into this
      * process, as it uses length of data */
-    char *umkey = ztrdup(pm->node.nam);
     int umlen = 0;
-    umkey = unmetafy(umkey,&umlen);
+    char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);
 
     key.dptr = umkey;
     key.dsize = umlen;
@@ -318,7 +318,7 @@ gdbmgetfn(Param pm)
          * can obtain data length to avoid using \0 */
         pm->u.str = metafy(content.dptr, content.dsize, META_ALLOC);
 
-        /* Free key */
+        /* Free key, restoring its original length */
         zsfree(umkey);
 
         /* Can return pointer, correctly saved inside hash */
@@ -358,17 +358,15 @@ gdbmsetfn(Param pm, char *val)
     /* Database */
     dbf = ((struct gsu_scalar_ext *)pm->gsu.s)->dbf;
     if (dbf) {
-        char *umkey = ztrdup(pm->node.nam);
         int umlen = 0;
-        umkey = unmetafy(umkey,&umlen);
+        char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);
 
         key.dptr = umkey;
         key.dsize = umlen;
 
         if (val) {
-            /* Unmetafy */
-            char *umval = ztrdup(val);
-            umval = unmetafy(umval,&umlen);
+            /* Unmetafy with exact zalloc size */
+            char *umval = unmetafy_zalloc(val,&umlen);
 
             /* Store */
             content.dptr = umval;
@@ -433,8 +431,7 @@ getgdbmnode(HashTable ht, const char *name)
 static void
 scangdbmkeys(HashTable ht, ScanFunc func, int flags)
 {
-    Param pm = NULL;
-    datum key, content;
+    datum key;
     GDBM_FILE dbf = ((struct gsu_scalar_ext *)ht->tmpdata)->dbf;
 
     /* Iterate keys adding them to hash, so
@@ -505,9 +502,8 @@ gdbmhashsetfn(Param pm, HashTable ht)
 	    v.pm = (Param) hn;
 
             /* Unmetafy key */
-            char *umkey = ztrdup(v.pm->node.nam);
             int umlen = 0;
-            umkey = unmetafy(umkey,&umlen);
+            char *umkey = unmetafy_zalloc(v.pm->node.nam,&umlen);
 
 	    key.dptr = umkey;
 	    key.dsize = umlen;
@@ -515,15 +511,16 @@ gdbmhashsetfn(Param pm, HashTable ht)
 	    queue_signals();
 
             /* Unmetafy */
-            char *umval = ztrdup(getstrvalue(&v));
-            umval = unmetafy(umval,&umlen);
+            char *umval = unmetafy_zalloc(getstrvalue(&v),&umlen);
 
             /* Store */
 	    content.dptr = umval;
 	    content.dsize = umlen;
 	    (void)gdbm_store(dbf, key, content, GDBM_REPLACE);	
 
-            /* Free */
+            /* Free - thanks to unmetafy_zalloc size of
+             * the strings is exact zalloc size - can
+             * pass to zsfree */
             zsfree(umval);
             zsfree(umkey);
 
@@ -739,3 +736,33 @@ static int remove_tied_name( const char *name ) {
 
     return 0;
 }
+
+/*
+ * Unmetafy that:
+ * - duplicates bufer to work on it,
+ * - does zalloc of exact size for the new string,
+ * - restores work buffer to original content, to restore strlen
+ *
+ * No zsfree()-confusing string will be produced.
+ */
+char *unmetafy_zalloc(const char *to_copy, int *new_len) {
+    char *work, *to_return;
+    int my_new_len = 0;
+
+    work = ztrdup(to_copy);
+    work = unmetafy(work,&my_new_len);
+
+    if (new_len)
+        *new_len = my_new_len;
+
+    /* This string can be correctly zsfree()-d */
+    to_return = (char *) zalloc((my_new_len+1)*sizeof(char));
+    memcpy(to_return, work, sizeof(char)*my_new_len); // memcpy handles $'\0'
+    to_return[my_new_len]='\0';
+
+    /* Restore original strlen and correctly free */
+    strcpy(work, to_copy);
+    zsfree(work);
+
+    return to_return;
+}
diff --git a/Doc/Zsh/mod_db_gdbm.yo b/Doc/Zsh/mod_db_gdbm.yo
index 9097429..699e9ab 100644
--- a/Doc/Zsh/mod_db_gdbm.yo
+++ b/Doc/Zsh/mod_db_gdbm.yo
@@ -43,6 +43,17 @@ local scope (function) ends.  Note that a readonly parameter may not be
 explicitly unset, so the only way to unset a global parameter created with
 `tt(ztie -r)' is to use `tt(zuntie -u)'.
 )
+findex(zgdbmpath)
+cindex(database file path, reading)
+item(tt(zgdbmpath) var(parametername))(
+Put path to database file assigned to var(parametername) into tt(REPLY)
+scalar.
+)
+findex(zgdbm_tied)
+cindex(database tied arrays, enumerating)
+item(tt(zgdbm_tied))(
+Array holding names of all tied parameters.
+)
 enditem()
 
 The fields of an associative array tied to GDBM are neither cached nor
diff --git a/Src/Modules/db_gdbm.c b/Src/Modules/db_gdbm.c
index 310e329..0a16540 100644
--- a/Src/Modules/db_gdbm.c
+++ b/Src/Modules/db_gdbm.c
@@ -6,6 +6,9 @@
  * Copyright (c) 2008 Clint Adams
  * All rights reserved.
  *
+ * Modifications copyright (c) 2017 Sebastian Gniazdowski
+ * All rights reserved.
+ *
  * Permission is hereby granted, without written agreement and without
  * license or royalty fees, to use, copy, modify, and distribute this
  * software and to distribute modified versions of this software for any
@@ -31,6 +34,15 @@
 #include "db_gdbm.mdh"
 #include "db_gdbm.pro"
 
+#ifndef PM_UPTODATE
+#define PM_UPTODATE     (1<<19) /* Parameter has up-to-date data (e.g. loaded from DB) */
+#endif
+
+static Param createhash( char *name, int flags );
+static int append_tied_name( const char *name );
+static int remove_tied_name( const char *name );
+char *unmetafy_zalloc(const char *to_copy, int *new_len);
+
 /*
  * Make sure we have all the bits I'm using for memory mapping, otherwise
  * I don't know what I'm doing.
@@ -41,8 +53,34 @@
 
 static char *backtype = "db/gdbm";
 
-static const struct gsu_scalar gdbm_gsu =
-{ gdbmgetfn, gdbmsetfn, gdbmunsetfn };
+/*
+ * Longer GSU structure, to carry GDBM_FILE of owning
+ * database. Every parameter (hash value) receives GSU
+ * pointer and thus also receives GDBM_FILE - this way
+ * parameters can access proper database.
+ *
+ * Main HashTable parameter has the same instance of
+ * the custom GSU struct in u.hash->tmpdata field.
+ * When database is closed, `dbf` field is set to NULL
+ * and hash values know to not access database when
+ * being unset (total purge at zuntie).
+ *
+ * When database closing is ended, custom GSU struct
+ * is freed. Only new ztie creates new custom GSU
+ * struct instance.
+ */
+
+struct gsu_scalar_ext {
+    struct gsu_scalar std;
+    GDBM_FILE dbf;
+    char *dbfile_path;
+};
+
+/* Source structure - will be copied to allocated one,
+ * with `dbf` filled. `dbf` allocation <-> gsu allocation. */
+static const struct gsu_scalar_ext gdbm_gsu_ext =
+{ { gdbmgetfn, gdbmsetfn, gdbmunsetfn }, 0, 0 };
+
 /**/
 static const struct gsu_hash gdbm_hash_gsu =
 { hashgetfn, gdbmhashsetfn, gdbmhashunsetfn };
@@ -50,6 +88,17 @@ static const struct gsu_hash gdbm_hash_gsu =
 static struct builtin bintab[] = {
     BUILTIN("ztie", 0, bin_ztie, 1, -1, 0, "d:f:r", NULL),
     BUILTIN("zuntie", 0, bin_zuntie, 1, -1, 0, "u", NULL),
+    BUILTIN("zgdbmpath", 0, bin_zgdbmpath, 1, -1, 0, "", NULL),
+};
+
+#define ROARRPARAMDEF(name, var) \
+    { name, PM_ARRAY | PM_READONLY, (void *) var, NULL,  NULL, NULL, NULL }
+
+/* Holds names of all tied parameters */
+char **zgdbm_tied;
+
+static struct paramdef patab[] = {
+    ROARRPARAMDEF( "zgdbm_tied", &zgdbm_tied ),
 };
 
 /**/
@@ -77,8 +126,7 @@ bin_ztie(char *nam, char **args, Options ops, UNUSED(int func))
     }
 
     /* Here should be a lookup of the backend type against
-     * a registry.
-     */
+     * a registry, if generam DB mechanism is to be added */
     if (strcmp(OPT_ARG(ops, 'd'), backtype) != 0) {
         zwarnnam(nam, "unsupported backend type `%s'", OPT_ARG(ops, 'd'));
 	return 1;
@@ -92,7 +140,8 @@ bin_ztie(char *nam, char **args, Options ops, UNUSED(int func))
 	/*
 	 * Unset any existing parameter.  Note there's no implicit
 	 * "local" here, but if the existing parameter is local
-	 * that will be reflected in the new one.
+	 * then new parameter will be also local without following
+         * unset.
 	 *
 	 * We need to do this before attempting to open the DB
 	 * in case this variable is already tied to a DB.
@@ -106,15 +155,15 @@ bin_ztie(char *nam, char **args, Options ops, UNUSED(int func))
     }
 
     dbf = gdbm_open(resource_name, 0, read_write, 0666, 0);
-    if(dbf)
-	addmodulefd(gdbm_fdesc(dbf), FDT_INTERNAL);
-    else {
+    if(dbf) {
+	addmodulefd(gdbm_fdesc(dbf), FDT_MODULE);
+        append_tied_name(pmname);
+    } else {
 	zwarnnam(nam, "error opening database file %s", resource_name);
 	return 1;
     }
 
-    if (!(tied_param = createspecialhash(pmname, &getgdbmnode, &scangdbmkeys,
-					 pmflags))) {
+    if (!(tied_param = createhash(pmname, pmflags))) {
         zwarnnam(nam, "cannot create the requested parameter %s", pmname);
 	fdtable[gdbm_fdesc(dbf)] = FDT_UNUSED;
 	gdbm_close(dbf);
@@ -122,8 +171,23 @@ bin_ztie(char *nam, char **args, Options ops, UNUSED(int func))
     }
 
     tied_param->gsu.h = &gdbm_hash_gsu;
-    tied_param->u.hash->tmpdata = (void *)dbf;
 
+    /* Allocate parameter sub-gsu, fill dbf field. 
+     * dbf allocation is 1 to 1 accompanied by
+     * gsu_scalar_ext allocation. */
+
+    struct gsu_scalar_ext *dbf_carrier = (struct gsu_scalar_ext *) zalloc(sizeof(struct gsu_scalar_ext));
+    dbf_carrier->std = gdbm_gsu_ext.std;
+    dbf_carrier->dbf = dbf;
+    tied_param->u.hash->tmpdata = (void *)dbf_carrier;
+
+    /* Fill also file path field */
+    if (*resource_name != '/') {
+        /* Code copied from check_autoload() */
+        resource_name = zhtricat(metafy(zgetcwd(), -1, META_HEAPDUP), "/", resource_name);
+        resource_name = xsymlink(resource_name, 1);
+    }
+    dbf_carrier->dbfile_path = ztrdup(resource_name);
     return 0;
 }
 
@@ -162,6 +226,53 @@ bin_zuntie(char *nam, char **args, Options ops, UNUSED(int func))
 }
 
 /**/
+static int
+bin_zgdbmpath(char *nam, char **args, Options ops, UNUSED(int func))
+{
+    Param pm;
+    char *pmname;
+
+    pmname = *args;
+
+    if (!pmname) {
+        zwarnnam(nam, "parameter name (whose path is to be written to $REPLY) is required");
+        return 1;
+    }
+
+    pm = (Param) paramtab->getnode(paramtab, pmname);
+    if(!pm) {
+        zwarnnam(nam, "no such parameter: %s", pmname);
+        return 1;
+    }
+
+    if (pm->gsu.h != &gdbm_hash_gsu) {
+        zwarnnam(nam, "not a tied gdbm parameter: %s", pmname);
+        return 1;
+    }
+
+    /* Paranoia, it *will* be always set */
+    if (((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbfile_path) {
+        setsparam("REPLY", ztrdup(((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbfile_path));
+    } else {
+        setsparam("REPLY", ztrdup(""));
+    }
+
+    return 0;
+}
+
+/*
+ * The param is actual param in hash â?? always, because
+ * getgdbmnode creates every new key seen. However, it
+ * might be not PM_UPTODATE - which means that database
+ * wasn't yet queried.
+ *
+ * It will be left in this state if database doesn't
+ * contain such key. That might be a drawback, maybe
+ * setting to empty value has sense, as no other writer
+ * can exist. This would remove subtle hcalloc(1) leak.
+ */
+
+/**/
 static char *
 gdbmgetfn(Param pm)
 {
@@ -169,18 +280,56 @@ gdbmgetfn(Param pm)
     int ret;
     GDBM_FILE dbf;
 
-    key.dptr = pm->node.nam;
-    key.dsize = strlen(key.dptr);
+    /* Key already retrieved? There is no sense of asking the
+     * database again, because:
+     * - there can be only multiple readers
+     * - so, no writer + reader use is allowed
+     *
+     * Thus:
+     * - if we are writers, we for sure have newest copy of data
+     * - if we are readers, we for sure have newest copy of data
+     */
+    if ( pm->node.flags & PM_UPTODATE ) {
+        return pm->u.str ? pm->u.str : (char *) hcalloc(1);
+    }
+
+    /* Unmetafy key. GDBM fits nice into this
+     * process, as it uses length of data */
+    int umlen = 0;
+    char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);
+
+    key.dptr = umkey;
+    key.dsize = umlen;
+
+    dbf = ((struct gsu_scalar_ext *)pm->gsu.s)->dbf;
+
+    if((ret = gdbm_exists(dbf, key))) {
+        /* We have data â?? store it, return it */
+        pm->node.flags |= PM_UPTODATE;
 
-    dbf = (GDBM_FILE)(pm->u.hash->tmpdata);
-    ret = gdbm_exists(dbf, key);
-    if(ret) {
         content = gdbm_fetch(dbf, key);
-    } else {
-        content.dptr = dupstring("");
+
+        /* Ensure there's no leak */
+        if (pm->u.str) {
+            zsfree(pm->u.str);
+        }
+
+        /* Metafy returned data. All fits - metafy
+         * can obtain data length to avoid using \0 */
+        pm->u.str = metafy(content.dptr, content.dsize, META_ALLOC);
+
+        /* Free key, restoring its original length */
+        zsfree(umkey);
+
+        /* Can return pointer, correctly saved inside hash */
+        return pm->u.str;
     }
 
-    return content.dptr;
+    /* Free key */
+    zsfree(umkey);
+
+    /* Can this be "" ? */
+    return (char *) hcalloc(1);
 }
 
 /**/
@@ -190,78 +339,126 @@ gdbmsetfn(Param pm, char *val)
     datum key, content;
     GDBM_FILE dbf;
 
-    key.dptr = pm->node.nam;
-    key.dsize = strlen(key.dptr);
-    content.dptr = val;
-    content.dsize = strlen(content.dptr);
+    /* Set is done on parameter and on database.
+     * See the allowed workers / readers comment
+     * at gdbmgetfn() */
+
+    /* Parameter */
+    if (pm->u.str) {
+        zsfree(pm->u.str);
+        pm->u.str = NULL;
+        pm->node.flags &= ~(PM_UPTODATE);
+    }
+
+    if (val) {
+        pm->u.str = ztrdup(val);
+        pm->node.flags |= PM_UPTODATE;
+    }
+
+    /* Database */
+    dbf = ((struct gsu_scalar_ext *)pm->gsu.s)->dbf;
+    if (dbf) {
+        int umlen = 0;
+        char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);
 
-    dbf = (GDBM_FILE)(pm->u.hash->tmpdata);
-    (void)gdbm_store(dbf, key, content, GDBM_REPLACE);
+        key.dptr = umkey;
+        key.dsize = umlen;
+
+        if (val) {
+            /* Unmetafy with exact zalloc size */
+            char *umval = unmetafy_zalloc(val,&umlen);
+
+            /* Store */
+            content.dptr = umval;
+            content.dsize = umlen;
+            (void)gdbm_store(dbf, key, content, GDBM_REPLACE);
+
+            /* Free */
+            zsfree(umval);
+        } else {
+            (void)gdbm_delete(dbf, key);
+        }
+
+        /* Free key */
+        zsfree(umkey);
+    }
 }
 
 /**/
 static void
 gdbmunsetfn(Param pm, UNUSED(int um))
 {
-    datum key;
-    GDBM_FILE dbf;
-
-    key.dptr = pm->node.nam;
-    key.dsize = strlen(key.dptr);
-
-    dbf = (GDBM_FILE)(pm->u.hash->tmpdata);
-    (void)gdbm_delete(dbf, key);
+    /* Set with NULL */
+    gdbmsetfn(pm, NULL);
 }
 
 /**/
 static HashNode
 getgdbmnode(HashTable ht, const char *name)
 {
-    int len;
-    char *nameu;
-    Param pm = NULL;
-
-    nameu = dupstring(name);
-    unmetafy(nameu, &len);
-
-    pm = (Param) hcalloc(sizeof(struct param));
-    pm->node.nam = nameu;
-    pm->node.flags = PM_SCALAR;
-    pm->gsu.s = &gdbm_gsu;
-    pm->u.hash = ht;
+    HashNode hn = gethashnode2( ht, name );
+    Param val_pm = (Param) hn;
+
+    /* Entry for key doesn't exist? Create it now,
+     * it will be interfacing between the database
+     * and Zsh - through special gdbm_gsu. So, any
+     * seen key results in new interfacing parameter.
+     *
+     * Previous code was returning heap arena Param
+     * that wasn't actually added to the hash. It was
+     * plainly name / database-key holder. Here we
+     * add the Param to its hash, it is not PM_UPTODATE.
+     * It will be loaded from database *and filled*
+     * or left in that state if the database doesn't
+     * contain it.
+     *
+     * No heap arena memory is used, memory usage is
+     * now limited - by number of distinct keys seen,
+     * not by number of key *uses*.
+     * */
+
+    if ( ! val_pm ) {
+        val_pm = (Param) zshcalloc( sizeof (*val_pm) );
+        val_pm->node.flags = PM_SCALAR | PM_HASHELEM; /* no PM_UPTODATE */
+        val_pm->gsu.s = (GsuScalar) ht->tmpdata;
+        ht->addnode( ht, ztrdup( name ), val_pm ); // sets pm->node.nam
+    }
 
-    return &pm->node;
+    return (HashNode) val_pm;
 }
 
 /**/
 static void
 scangdbmkeys(HashTable ht, ScanFunc func, int flags)
 {
-    Param pm = NULL;
-    datum key, content;
-    GDBM_FILE dbf = (GDBM_FILE)(ht->tmpdata);
-
-    pm = (Param) hcalloc(sizeof(struct param));
-
-    pm->node.flags = PM_SCALAR;
-    pm->gsu.s = &nullsetscalar_gsu;
+    datum key;
+    GDBM_FILE dbf = ((struct gsu_scalar_ext *)ht->tmpdata)->dbf;
 
+    /* Iterate keys adding them to hash, so
+     * we have Param to use in `func` */
     key = gdbm_firstkey(dbf);
 
     while(key.dptr) {
-	content = gdbm_fetch(dbf, key);
-
-	pm->node.nam = key.dptr;
-	pm->u.str = content.dptr;
-	pm->gsu.s = &nullsetscalar_gsu;
+        /* This returns database-interfacing Param,
+         * it will return u.str or first fetch data
+         * if not PM_UPTODATE (newly created) */
+        char *zkey = ztrduppfx(key.dptr, key.dsize);
+        HashNode hn = getgdbmnode(ht, zkey);
+        zsfree( zkey );
 
-	func(&pm->node, flags);
+	func(hn, flags);
 
+        /* Iterate - no problem as interfacing Param
+         * will do at most only fetches, not stores */
         key = gdbm_nextkey(dbf, key);
     }
 
 }
 
+/*
+ * Replace database with new hash
+ */
+
 /**/
 static void
 gdbmhashsetfn(Param pm, HashTable ht)
@@ -274,7 +471,7 @@ gdbmhashsetfn(Param pm, HashTable ht)
     if (!pm->u.hash || pm->u.hash == ht)
 	return;
 
-    if (!(dbf = (GDBM_FILE)(pm->u.hash->tmpdata)))
+    if (!(dbf = ((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbf))
 	return;
 
     key = gdbm_firstkey(dbf);
@@ -292,6 +489,9 @@ gdbmhashsetfn(Param pm, HashTable ht)
     if (!ht)
 	return;
 
+     /* Put new strings into database, waiting
+      * for their interfacing-Params to be created */
+
     for (i = 0; i < ht->hsize; i++)
 	for (hn = ht->nodes[i]; hn; hn = hn->next) {
 	    struct value v;
@@ -301,16 +501,29 @@ gdbmhashsetfn(Param pm, HashTable ht)
 	    v.arr = NULL;
 	    v.pm = (Param) hn;
 
-	    key.dptr = v.pm->node.nam;
-	    key.dsize = strlen(key.dptr);
+            /* Unmetafy key */
+            int umlen = 0;
+            char *umkey = unmetafy_zalloc(v.pm->node.nam,&umlen);
+
+	    key.dptr = umkey;
+	    key.dsize = umlen;
 
 	    queue_signals();
 
-	    content.dptr = getstrvalue(&v);
-	    content.dsize = strlen(content.dptr);
+            /* Unmetafy */
+            char *umval = unmetafy_zalloc(getstrvalue(&v),&umlen);
 
+            /* Store */
+	    content.dptr = umval;
+	    content.dsize = umlen;
 	    (void)gdbm_store(dbf, key, content, GDBM_REPLACE);	
 
+            /* Free - thanks to unmetafy_zalloc size of
+             * the strings is exact zalloc size - can
+             * pass to zsfree */
+            zsfree(umval);
+            zsfree(umkey);
+
 	    unqueue_signals();
 	}
 }
@@ -319,15 +532,19 @@ gdbmhashsetfn(Param pm, HashTable ht)
 static void
 gdbmuntie(Param pm)
 {
-    GDBM_FILE dbf = (GDBM_FILE)(pm->u.hash->tmpdata);
+    GDBM_FILE dbf = ((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbf;
     HashTable ht = pm->u.hash;
 
     if (dbf) { /* paranoia */
 	fdtable[gdbm_fdesc(dbf)] = FDT_UNUSED;
-	gdbm_close(dbf);
-    }
+        gdbm_close(dbf);
 
-    ht->tmpdata = NULL;
+        /* Let hash fields know there's no backend */
+        ((struct gsu_scalar_ext *)ht->tmpdata)->dbf = NULL;
+
+        /* Remove from list of tied parameters */
+        remove_tied_name(pm->node.nam);
+    }
 
     /* for completeness ... createspecialhash() should have an inverse */
     ht->getnode = ht->getnode2 = gethashnode2;
@@ -342,8 +559,20 @@ static void
 gdbmhashunsetfn(Param pm, UNUSED(int exp))
 {
     gdbmuntie(pm);
-    /* hash table is now normal, so proceed normally... */
+
+    /* Remember custom GSU structure assigned to
+     * u.hash->tmpdata before hash gets deleted */
+    struct gsu_scalar_ext * gsu_ext = pm->u.hash->tmpdata;
+
+    /* Uses normal unsetter. Will delete all owned
+     * parameters and also hashtable. */
     pm->gsu.h->setfn(pm, NULL);
+
+    /* Don't need custom GSU structure with its
+     * GDBM_FILE pointer anymore */
+    zsfree( gsu_ext->dbfile_path );
+    zfree( gsu_ext, sizeof(struct gsu_scalar_ext));
+
     pm->node.flags |= PM_UNSET;
 }
 
@@ -355,7 +584,7 @@ static struct features module_features = {
     bintab, sizeof(bintab)/sizeof(*bintab),
     NULL, 0,
     NULL, 0,
-    NULL, 0,
+    patab, sizeof(patab)/sizeof(*patab),
     0
 };
 
@@ -385,6 +614,7 @@ enables_(Module m, int **enables)
 int
 boot_(UNUSED(Module m))
 {
+    zgdbm_tied = zshcalloc((1) * sizeof(char *));
     return 0;
 }
 
@@ -392,6 +622,7 @@ boot_(UNUSED(Module m))
 int
 cleanup_(Module m)
 {
+    /* This frees `zgdbm_tied` */
     return setfeatureenables(m, &module_features, NULL);
 }
 
@@ -401,3 +632,137 @@ finish_(UNUSED(Module m))
 {
     return 0;
 }
+
+/*********************
+ * Utility functions *
+ *********************/
+
+static Param createhash( char *name, int flags ) {
+    Param pm;
+    HashTable ht;
+
+    pm = createparam(name, PM_SPECIAL | PM_HASHED);
+    if (!pm) {
+        return NULL;
+    }
+
+    if (pm->old)
+	pm->level = locallevel;
+
+    /* This creates standard hash. */
+    ht = pm->u.hash = newparamtable(32, name);
+    if (!pm->u.hash) {
+        paramtab->removenode(paramtab, name);
+        paramtab->freenode(&pm->node);
+        zwarnnam(name, "Out of memory when allocating hash");
+    }
+
+    /* These provide special features */
+    ht->getnode = ht->getnode2 = getgdbmnode;
+    ht->scantab = scangdbmkeys;
+
+    return pm;
+}
+
+/*
+ * Adds parameter name to `zgdbm_tied`
+ */
+
+static int append_tied_name( const char *name ) {
+    int old_len = arrlen(zgdbm_tied);
+    char **new_zgdbm_tied = zshcalloc( (old_len+2) * sizeof(char *));
+
+    /* Copy */
+    char **p = zgdbm_tied;
+    char **dst = new_zgdbm_tied;
+    while (*p) {
+        *dst++ = *p++;
+    }
+
+    /* Append new one */
+    *dst = ztrdup(name);
+
+    /* Substitute, free old one */
+    zfree(zgdbm_tied, sizeof(char *) * (old_len + 1));
+    zgdbm_tied = new_zgdbm_tied;
+
+    return 0;
+}
+
+/*
+ * Removes parameter name from `zgdbm_tied`
+ */
+
+static int remove_tied_name( const char *name ) {
+    int old_len = arrlen(zgdbm_tied);
+
+    /* Two stage, to always have arrlen() == zfree-size - 1.
+     * Could do allocation and revert when `not found`, but
+     * what would be better about that. */
+
+    /* Find one to remove */
+    char **p = zgdbm_tied;
+    while (*p) {
+        if (0==strcmp(name,*p)) {
+            break;
+        }
+        p++;
+    }
+
+    /* Copy x+1 to x */
+    while (*p) {
+        *p=*(p+1);
+        p++;
+    }
+
+    /* Second stage. Size changed? Only old_size-1
+     * change is possible, but.. paranoia way */
+    int new_len = arrlen(zgdbm_tied);
+    if (new_len != old_len) {
+        char **new_zgdbm_tied = zshcalloc((new_len+1) * sizeof(char *));
+
+        /* Copy */
+        p = zgdbm_tied;
+        char **dst = new_zgdbm_tied;
+        while (*p) {
+            *dst++ = *p++;
+        }
+        *dst = NULL;
+
+        /* Substitute, free old one */
+        zfree(zgdbm_tied, sizeof(char *) * (old_len + 1));
+        zgdbm_tied = new_zgdbm_tied;
+    }
+
+    return 0;
+}
+
+/*
+ * Unmetafy that:
+ * - duplicates bufer to work on it,
+ * - does zalloc of exact size for the new string,
+ * - restores work buffer to original content, to restore strlen
+ *
+ * No zsfree()-confusing string will be produced.
+ */
+char *unmetafy_zalloc(const char *to_copy, int *new_len) {
+    char *work, *to_return;
+    int my_new_len = 0;
+
+    work = ztrdup(to_copy);
+    work = unmetafy(work,&my_new_len);
+
+    if (new_len)
+        *new_len = my_new_len;
+
+    /* This string can be correctly zsfree()-d */
+    to_return = (char *) zalloc((my_new_len+1)*sizeof(char));
+    memcpy(to_return, work, sizeof(char)*my_new_len); // memcpy handles $'\0'
+    to_return[my_new_len]='\0';
+
+    /* Restore original strlen and correctly free */
+    strcpy(work, to_copy);
+    zsfree(work);
+
+    return to_return;
+}
diff --git a/Src/Modules/db_gdbm.mdd b/Src/Modules/db_gdbm.mdd
index ce7926b..210c221 100644
--- a/Src/Modules/db_gdbm.mdd
+++ b/Src/Modules/db_gdbm.mdd
@@ -7,6 +7,6 @@ fi
 '
 load=no
 
-autofeatures="b:ztie b:zuntie"
+autofeatures="b:ztie b:zuntie b:zgdbmpath p:zgdbm_tied"
 
 objects="db_gdbm.o"
diff --git a/Test/V11db_gdbm.ztst b/Test/V11db_gdbm.ztst
new file mode 100644
index 0000000..a1076dc
--- /dev/null
+++ b/Test/V11db_gdbm.ztst
@@ -0,0 +1,285 @@
+# Tests for the zsh/param/private module
+
+%prep
+
+ modname="zsh/db/gdbm"
+ dbfile=db.gdbm
+ if ! zmodload $modname ; then
+   ZTST_unimplemented="can't load $modname module for testing"
+ fi
+ rm -f db.gdbm
+
+%test
+
+ (zmodload -u $modname && zmodload $modname)
+0:unload and reload the module without crashing
+
+ ztie -d db/gdbm -f $dbfile dbase
+ zuntie dbase
+0:create the database
+
+ ztie -r -d db/gdbm -f $dbfile dbase
+ zuntie -u dbase
+0:open the database read-only
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[testkey]=testdata
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[testkey]
+ zuntie -u dbase
+0:store key in database
+>testdata
+
+ ztie -d db/gdbm -f $dbfile dbase2
+ unset 'dbase2[testkey]'
+ zuntie dbase2
+ ztie -d db/gdbm -f $dbfile dbase
+ echo $dbase[testkey]
+ zuntie dbase
+0:remove key from database (different variables)
+>
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[testkey]=testdata
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[testkey]
+ zuntie -u dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ unset 'dbase[testkey]'
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[testkey]
+ zuntie -u dbase
+0:store & remove key from database (the same variables)
+>testdata
+>
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[testkey]=testdata
+ dbase[testkey2]=$dbase[testkey]
+ dbase[testkey3]=$dbase[testkey]x$dbase[testkey2]
+ zuntie dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ echo $dbase[testkey]
+ echo $dbase[testkey2]
+ echo $dbase[testkey3]
+ zuntie dbase
+0:store 2 keys fetching 1st
+>testdata
+>testdata
+>testdataxtestdata
+
+ ztie -d db/gdbm -f $dbfile dbase
+ val=$dbase[testkey2]
+ unset 'dbase[testkey2]'
+ echo $val
+ zuntie dbase
+0:unset key that was fetched
+>testdata
+
+ ztie -r -d db/gdbm -f $dbfile dbase
+ print -rl -- "${(kv)dbase[@]}"
+ zuntie -u dbase
+0:scan read-only tied hash
+>testkey
+>testdata
+>testkey3
+>testdataxtestdata
+
+ ztie -d db/gdbm -f $dbfile dbase
+ local -a arr
+ arr=( "${(kv)dbase[@]}" )
+ print -rl -- "${arr[@]}"
+ zuntie dbase
+0:different scan, also read-write mode
+>testkey
+>testdata
+>testkey3
+>testdataxtestdata
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase=( a b c d )
+ zuntie dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ print -rl -- "${(kv)dbase[@]}"
+ zuntie dbase
+0:replace hash / database, scan
+>c
+>d
+>a
+>b
+
+ ztie -d db/gdbm -f $dbfile dbase
+ local -a arr
+ arr=( "${dbase[@]}" )
+ print -rl -- "${arr[@]}"
+ zuntie dbase
+0:scan with no (kv)
+>d
+>b
+
+ ztie -d db/gdbm -f $dbfile dbase
+ print -rl -- "${(k)dbase[@]}"
+ zuntie dbase
+0:scan with keys only (k)
+>c
+>a
+
+ ztie -d db/gdbm -f $dbfile dbase
+ print -rl -- "${(v)dbase[@]}"
+ zuntie dbase
+0:scan with keys only explicit (v)
+>d
+>b
+
+ rm -f $dbfile
+ ztie -r -d db/gdbm -f $dbfile dbase 2>/dev/null
+1:read-only open non-existent database
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase+=( a b )
+ echo $dbase[a]
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[a]
+ print -rl -- "${(kv)dbase[@]}"
+ zuntie -u dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase+=( c d )
+ echo $dbase[a]
+ echo $dbase[c]
+ print -rl -- "${(kv)dbase[@]}"
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[a]
+ echo $dbase[c]
+ print -rl -- "${(kv)dbase[@]}"
+ zuntie -u dbase
+0:Append with +=( ), also with existing data, also (kv) scan
+>b
+>b
+>a
+>b
+>b
+>d
+>c
+>d
+>a
+>b
+>b
+>d
+>c
+>d
+>a
+>b
+
+ ztie -d db/gdbm -f $dbfile dbase
+ echo ${(t)dbase}
+ zuntie dbase
+0:Type of tied parameter
+>association-special
+
+ typeset -ga dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ echo ${(t)dbase}
+ zuntie dbase
+0:Type of tied parameter, with preceding unset
+>association-special
+
+ local -a dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ echo ${(t)dbase}
+ zuntie dbase
+0:Type of tied parameter, with local parameter already existing
+>association-local-special
+
+ local -a dbase
+ dbase=( fromarray )
+ () {
+     local -a dbase
+     ztie -d db/gdbm -f $dbfile dbase
+     echo ${(t)dbase}
+     zuntie dbase
+ }
+ echo $dbase[1]
+ ztie -d db/gdbm -f $dbfile dbase2
+ echo "Can connect, so untie happened:" $dbase2[a]
+ zuntie dbase2
+0:Test of automatic untie (use of local scope) and of scoping
+>association-local-special
+>fromarray
+>Can connect, so untie happened: b
+
+ echo $zgdbm_tied ${#zgdbm_tied}
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $zgdbm_tied ${#zgdbm_tied}
+ ztie -d db/gdbm -f ${dbfile}2 dbase2
+ echo $zgdbm_tied ${#zgdbm_tied}
+ zuntie -u dbase
+ echo $zgdbm_tied ${#zgdbm_tied}
+ zuntie dbase2
+ echo $zgdbm_tied ${#zgdbm_tied}
+0:zgdbm_tied parameter
+>0
+>dbase 1
+>dbase dbase2 2
+>dbase2 1
+>0
+
+ unset zgdbm_tied 2>/dev/null
+1:unset of read-only zgdbm_tied parameter
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[漢�]=漢�
+ echo $dbase[漢�]
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[漢�]
+ zuntie -u dbase
+0:Unicode test
+>漢�
+>漢�
+
+ key="ab"$'\0'"ef"
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[$key]=value
+ echo $dbase[$key]
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ echo $dbase[$key]
+ zuntie -u dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase[$key]=$key
+ zuntie dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ [[ "$dbase[$key]" = "$key" ]] && echo correct
+ zuntie dbase
+0:Metafication of $'\0'
+>value
+>value
+>correct
+
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase=( 漢� 漢� )
+ echo $dbase[漢�]
+ zuntie dbase
+ ztie -d db/gdbm -f $dbfile dbase
+ echo $dbase[漢�]
+ zuntie dbase
+ key="ab"$'\0'"ef"
+ ztie -d db/gdbm -f $dbfile dbase
+ dbase+=( $key $key )
+ zuntie dbase
+ ztie -r -d db/gdbm -f $dbfile dbase
+ [[ "$dbase[$key]" = "$key" ]] && echo correct
+ zuntie -u dbase
+0:Unicode & metafication test, different hash access
+>漢�
+>漢�
+>correct
+
+%clean
+
+  rm -f ${dbfile}*
/*
 * db_gdbm.c - bindings for gdbm
 *
 * This file is part of zsh, the Z shell.
 *
 * Copyright (c) 2008 Clint Adams
 * All rights reserved.
 *
 * Modifications copyright (c) 2017 Sebastian Gniazdowski
 * All rights reserved.
 *
 * Permission is hereby granted, without written agreement and without
 * license or royalty fees, to use, copy, modify, and distribute this
 * software and to distribute modified versions of this software for any
 * purpose, provided that the above copyright notice and the following
 * two paragraphs appear in all copies of this software.
 *
 * In no event shall Clint Adams or the Zsh Development
 * Group be liable to any party for direct, indirect, special, incidental, or
 * consequential damages arising out of the use of this software and its
 * documentation, even if Peter Stephenson, Sven Wischnowsky and the Zsh
 * Development Group have been advised of the possibility of such damage.
 *
 * Clint Adams and the Zsh Development Group
 * specifically disclaim any warranties, including, but not limited to, the
 * implied warranties of merchantability and fitness for a particular purpose.
 * The software provided hereunder is on an "as is" basis, and Peter
 * Stephenson, Sven Wischnowsky and the Zsh Development Group have no
 * obligation to provide maintenance, support, updates, enhancements, or
 * modifications.
 *
 */

#include "db_gdbm.mdh"
#include "db_gdbm.pro"

#ifndef PM_UPTODATE
#define PM_UPTODATE     (1<<19) /* Parameter has up-to-date data (e.g. loaded from DB) */
#endif

static Param createhash( char *name, int flags );
static int append_tied_name( const char *name );
static int remove_tied_name( const char *name );
char *unmetafy_zalloc(const char *to_copy, int *new_len);

/*
 * Make sure we have all the bits I'm using for memory mapping, otherwise
 * I don't know what I'm doing.
 */
#if defined(HAVE_GDBM_H) && defined(HAVE_GDBM_OPEN)

#include <gdbm.h>

static char *backtype = "db/gdbm";

/*
 * Longer GSU structure, to carry GDBM_FILE of owning
 * database. Every parameter (hash value) receives GSU
 * pointer and thus also receives GDBM_FILE - this way
 * parameters can access proper database.
 *
 * Main HashTable parameter has the same instance of
 * the custom GSU struct in u.hash->tmpdata field.
 * When database is closed, `dbf` field is set to NULL
 * and hash values know to not access database when
 * being unset (total purge at zuntie).
 *
 * When database closing is ended, custom GSU struct
 * is freed. Only new ztie creates new custom GSU
 * struct instance.
 */

struct gsu_scalar_ext {
    struct gsu_scalar std;
    GDBM_FILE dbf;
    char *dbfile_path;
};

/* Source structure - will be copied to allocated one,
 * with `dbf` filled. `dbf` allocation <-> gsu allocation. */
static const struct gsu_scalar_ext gdbm_gsu_ext =
{ { gdbmgetfn, gdbmsetfn, gdbmunsetfn }, 0, 0 };

/**/
static const struct gsu_hash gdbm_hash_gsu =
{ hashgetfn, gdbmhashsetfn, gdbmhashunsetfn };

static struct builtin bintab[] = {
    BUILTIN("ztie", 0, bin_ztie, 1, -1, 0, "d:f:r", NULL),
    BUILTIN("zuntie", 0, bin_zuntie, 1, -1, 0, "u", NULL),
    BUILTIN("zgdbmpath", 0, bin_zgdbmpath, 1, -1, 0, "", NULL),
};

#define ROARRPARAMDEF(name, var) \
    { name, PM_ARRAY | PM_READONLY, (void *) var, NULL,  NULL, NULL, NULL }

/* Holds names of all tied parameters */
char **zgdbm_tied;

static struct paramdef patab[] = {
    ROARRPARAMDEF( "zgdbm_tied", &zgdbm_tied ),
};

/**/
static int
bin_ztie(char *nam, char **args, Options ops, UNUSED(int func))
{
    char *resource_name, *pmname;
    GDBM_FILE dbf = NULL;
    int read_write = GDBM_SYNC, pmflags = PM_REMOVABLE;
    Param tied_param;

    if(!OPT_ISSET(ops,'d')) {
        zwarnnam(nam, "you must pass `-d %s'", backtype);
	return 1;
    }
    if(!OPT_ISSET(ops,'f')) {
        zwarnnam(nam, "you must pass `-f' with a filename", NULL);
	return 1;
    }
    if (OPT_ISSET(ops,'r')) {
	read_write |= GDBM_READER;
	pmflags |= PM_READONLY;
    } else {
	read_write |= GDBM_WRCREAT;
    }

    /* Here should be a lookup of the backend type against
     * a registry, if generam DB mechanism is to be added */
    if (strcmp(OPT_ARG(ops, 'd'), backtype) != 0) {
        zwarnnam(nam, "unsupported backend type `%s'", OPT_ARG(ops, 'd'));
	return 1;
    }

    resource_name = OPT_ARG(ops, 'f');
    pmname = *args;

    if ((tied_param = (Param)paramtab->getnode(paramtab, pmname)) &&
	!(tied_param->node.flags & PM_UNSET)) {
	/*
	 * Unset any existing parameter.  Note there's no implicit
	 * "local" here, but if the existing parameter is local
	 * then new parameter will be also local without following
         * unset.
	 *
	 * We need to do this before attempting to open the DB
	 * in case this variable is already tied to a DB.
	 *
	 * This can fail if the variable is readonly or restricted.
	 * We could call unsetparam() and check errflag instead
	 * of the return status.
	 */
	if (unsetparam_pm(tied_param, 0, 1))
	    return 1;
    }

    dbf = gdbm_open(resource_name, 0, read_write, 0666, 0);
    if(dbf) {
	addmodulefd(gdbm_fdesc(dbf), FDT_MODULE);
        append_tied_name(pmname);
    } else {
	zwarnnam(nam, "error opening database file %s", resource_name);
	return 1;
    }

    if (!(tied_param = createhash(pmname, pmflags))) {
        zwarnnam(nam, "cannot create the requested parameter %s", pmname);
	fdtable[gdbm_fdesc(dbf)] = FDT_UNUSED;
	gdbm_close(dbf);
	return 1;
    }

    tied_param->gsu.h = &gdbm_hash_gsu;

    /* Allocate parameter sub-gsu, fill dbf field. 
     * dbf allocation is 1 to 1 accompanied by
     * gsu_scalar_ext allocation. */

    struct gsu_scalar_ext *dbf_carrier = (struct gsu_scalar_ext *) zalloc(sizeof(struct gsu_scalar_ext));
    dbf_carrier->std = gdbm_gsu_ext.std;
    dbf_carrier->dbf = dbf;
    tied_param->u.hash->tmpdata = (void *)dbf_carrier;

    /* Fill also file path field */
    if (*resource_name != '/') {
        /* Code copied from check_autoload() */
        resource_name = zhtricat(metafy(zgetcwd(), -1, META_HEAPDUP), "/", resource_name);
        resource_name = xsymlink(resource_name, 1);
    }
    dbf_carrier->dbfile_path = ztrdup(resource_name);
    return 0;
}

/**/
static int
bin_zuntie(char *nam, char **args, Options ops, UNUSED(int func))
{
    Param pm;
    char *pmname;
    int ret = 0;

    for (pmname = *args; *args++; pmname = *args) {
	pm = (Param) paramtab->getnode(paramtab, pmname);
	if(!pm) {
	    zwarnnam(nam, "cannot untie %s", pmname);
	    ret = 1;
	    continue;
	}
	if (pm->gsu.h != &gdbm_hash_gsu) {
	    zwarnnam(nam, "not a tied gdbm hash: %s", pmname);
	    ret = 1;
	    continue;
	}

	queue_signals();
	if (OPT_ISSET(ops,'u'))
	    gdbmuntie(pm);	/* clear read-only-ness */
	if (unsetparam_pm(pm, 0, 1)) {
	    /* assume already reported */
	    ret = 1;
	}
	unqueue_signals();
    }

    return ret;
}

/**/
static int
bin_zgdbmpath(char *nam, char **args, Options ops, UNUSED(int func))
{
    Param pm;
    char *pmname;

    pmname = *args;

    if (!pmname) {
        zwarnnam(nam, "parameter name (whose path is to be written to $REPLY) is required");
        return 1;
    }

    pm = (Param) paramtab->getnode(paramtab, pmname);
    if(!pm) {
        zwarnnam(nam, "no such parameter: %s", pmname);
        return 1;
    }

    if (pm->gsu.h != &gdbm_hash_gsu) {
        zwarnnam(nam, "not a tied gdbm parameter: %s", pmname);
        return 1;
    }

    /* Paranoia, it *will* be always set */
    if (((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbfile_path) {
        setsparam("REPLY", ztrdup(((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbfile_path));
    } else {
        setsparam("REPLY", ztrdup(""));
    }

    return 0;
}

/*
 * The param is actual param in hash â?? always, because
 * getgdbmnode creates every new key seen. However, it
 * might be not PM_UPTODATE - which means that database
 * wasn't yet queried.
 *
 * It will be left in this state if database doesn't
 * contain such key. That might be a drawback, maybe
 * setting to empty value has sense, as no other writer
 * can exist. This would remove subtle hcalloc(1) leak.
 */

/**/
static char *
gdbmgetfn(Param pm)
{
    datum key, content;
    int ret;
    GDBM_FILE dbf;

    /* Key already retrieved? There is no sense of asking the
     * database again, because:
     * - there can be only multiple readers
     * - so, no writer + reader use is allowed
     *
     * Thus:
     * - if we are writers, we for sure have newest copy of data
     * - if we are readers, we for sure have newest copy of data
     */
    if ( pm->node.flags & PM_UPTODATE ) {
        return pm->u.str ? pm->u.str : (char *) hcalloc(1);
    }

    /* Unmetafy key. GDBM fits nice into this
     * process, as it uses length of data */
    int umlen = 0;
    char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);

    key.dptr = umkey;
    key.dsize = umlen;

    dbf = ((struct gsu_scalar_ext *)pm->gsu.s)->dbf;

    if((ret = gdbm_exists(dbf, key))) {
        /* We have data â?? store it, return it */
        pm->node.flags |= PM_UPTODATE;

        content = gdbm_fetch(dbf, key);

        /* Ensure there's no leak */
        if (pm->u.str) {
            zsfree(pm->u.str);
        }

        /* Metafy returned data. All fits - metafy
         * can obtain data length to avoid using \0 */
        pm->u.str = metafy(content.dptr, content.dsize, META_ALLOC);

        /* Free key, restoring its original length */
        zsfree(umkey);

        /* Can return pointer, correctly saved inside hash */
        return pm->u.str;
    }

    /* Free key */
    zsfree(umkey);

    /* Can this be "" ? */
    return (char *) hcalloc(1);
}

/**/
static void
gdbmsetfn(Param pm, char *val)
{
    datum key, content;
    GDBM_FILE dbf;

    /* Set is done on parameter and on database.
     * See the allowed workers / readers comment
     * at gdbmgetfn() */

    /* Parameter */
    if (pm->u.str) {
        zsfree(pm->u.str);
        pm->u.str = NULL;
        pm->node.flags &= ~(PM_UPTODATE);
    }

    if (val) {
        pm->u.str = ztrdup(val);
        pm->node.flags |= PM_UPTODATE;
    }

    /* Database */
    dbf = ((struct gsu_scalar_ext *)pm->gsu.s)->dbf;
    if (dbf) {
        int umlen = 0;
        char *umkey = unmetafy_zalloc(pm->node.nam,&umlen);

        key.dptr = umkey;
        key.dsize = umlen;

        if (val) {
            /* Unmetafy with exact zalloc size */
            char *umval = unmetafy_zalloc(val,&umlen);

            /* Store */
            content.dptr = umval;
            content.dsize = umlen;
            (void)gdbm_store(dbf, key, content, GDBM_REPLACE);

            /* Free */
            zsfree(umval);
        } else {
            (void)gdbm_delete(dbf, key);
        }

        /* Free key */
        zsfree(umkey);
    }
}

/**/
static void
gdbmunsetfn(Param pm, UNUSED(int um))
{
    /* Set with NULL */
    gdbmsetfn(pm, NULL);
}

/**/
static HashNode
getgdbmnode(HashTable ht, const char *name)
{
    HashNode hn = gethashnode2( ht, name );
    Param val_pm = (Param) hn;

    /* Entry for key doesn't exist? Create it now,
     * it will be interfacing between the database
     * and Zsh - through special gdbm_gsu. So, any
     * seen key results in new interfacing parameter.
     *
     * Previous code was returning heap arena Param
     * that wasn't actually added to the hash. It was
     * plainly name / database-key holder. Here we
     * add the Param to its hash, it is not PM_UPTODATE.
     * It will be loaded from database *and filled*
     * or left in that state if the database doesn't
     * contain it.
     *
     * No heap arena memory is used, memory usage is
     * now limited - by number of distinct keys seen,
     * not by number of key *uses*.
     * */

    if ( ! val_pm ) {
        val_pm = (Param) zshcalloc( sizeof (*val_pm) );
        val_pm->node.flags = PM_SCALAR | PM_HASHELEM; /* no PM_UPTODATE */
        val_pm->gsu.s = (GsuScalar) ht->tmpdata;
        ht->addnode( ht, ztrdup( name ), val_pm ); // sets pm->node.nam
    }

    return (HashNode) val_pm;
}

/**/
static void
scangdbmkeys(HashTable ht, ScanFunc func, int flags)
{
    datum key;
    GDBM_FILE dbf = ((struct gsu_scalar_ext *)ht->tmpdata)->dbf;

    /* Iterate keys adding them to hash, so
     * we have Param to use in `func` */
    key = gdbm_firstkey(dbf);

    while(key.dptr) {
        /* This returns database-interfacing Param,
         * it will return u.str or first fetch data
         * if not PM_UPTODATE (newly created) */
        char *zkey = ztrduppfx(key.dptr, key.dsize);
        HashNode hn = getgdbmnode(ht, zkey);
        zsfree( zkey );

	func(hn, flags);

        /* Iterate - no problem as interfacing Param
         * will do at most only fetches, not stores */
        key = gdbm_nextkey(dbf, key);
    }

}

/*
 * Replace database with new hash
 */

/**/
static void
gdbmhashsetfn(Param pm, HashTable ht)
{
    int i;
    HashNode hn;
    GDBM_FILE dbf;
    datum key, content;

    if (!pm->u.hash || pm->u.hash == ht)
	return;

    if (!(dbf = ((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbf))
	return;

    key = gdbm_firstkey(dbf);
    while (key.dptr) {
	queue_signals();
	(void)gdbm_delete(dbf, key);
	free(key.dptr);
	unqueue_signals();
	key = gdbm_firstkey(dbf);
    }

    /* just deleted everything, clean up */
    (void)gdbm_reorganize(dbf);

    if (!ht)
	return;

     /* Put new strings into database, waiting
      * for their interfacing-Params to be created */

    for (i = 0; i < ht->hsize; i++)
	for (hn = ht->nodes[i]; hn; hn = hn->next) {
	    struct value v;

	    v.isarr = v.flags = v.start = 0;
	    v.end = -1;
	    v.arr = NULL;
	    v.pm = (Param) hn;

            /* Unmetafy key */
            int umlen = 0;
            char *umkey = unmetafy_zalloc(v.pm->node.nam,&umlen);

	    key.dptr = umkey;
	    key.dsize = umlen;

	    queue_signals();

            /* Unmetafy */
            char *umval = unmetafy_zalloc(getstrvalue(&v),&umlen);

            /* Store */
	    content.dptr = umval;
	    content.dsize = umlen;
	    (void)gdbm_store(dbf, key, content, GDBM_REPLACE);	

            /* Free - thanks to unmetafy_zalloc size of
             * the strings is exact zalloc size - can
             * pass to zsfree */
            zsfree(umval);
            zsfree(umkey);

	    unqueue_signals();
	}
}

/**/
static void
gdbmuntie(Param pm)
{
    GDBM_FILE dbf = ((struct gsu_scalar_ext *)pm->u.hash->tmpdata)->dbf;
    HashTable ht = pm->u.hash;

    if (dbf) { /* paranoia */
	fdtable[gdbm_fdesc(dbf)] = FDT_UNUSED;
        gdbm_close(dbf);

        /* Let hash fields know there's no backend */
        ((struct gsu_scalar_ext *)ht->tmpdata)->dbf = NULL;

        /* Remove from list of tied parameters */
        remove_tied_name(pm->node.nam);
    }

    /* for completeness ... createspecialhash() should have an inverse */
    ht->getnode = ht->getnode2 = gethashnode2;
    ht->scantab = NULL;

    pm->node.flags &= ~(PM_SPECIAL|PM_READONLY);
    pm->gsu.h = &stdhash_gsu;
}

/**/
static void
gdbmhashunsetfn(Param pm, UNUSED(int exp))
{
    gdbmuntie(pm);

    /* Remember custom GSU structure assigned to
     * u.hash->tmpdata before hash gets deleted */
    struct gsu_scalar_ext * gsu_ext = pm->u.hash->tmpdata;

    /* Uses normal unsetter. Will delete all owned
     * parameters and also hashtable. */
    pm->gsu.h->setfn(pm, NULL);

    /* Don't need custom GSU structure with its
     * GDBM_FILE pointer anymore */
    zsfree( gsu_ext->dbfile_path );
    zfree( gsu_ext, sizeof(struct gsu_scalar_ext));

    pm->node.flags |= PM_UNSET;
}

#else
# error no gdbm
#endif /* have gdbm */

static struct features module_features = {
    bintab, sizeof(bintab)/sizeof(*bintab),
    NULL, 0,
    NULL, 0,
    patab, sizeof(patab)/sizeof(*patab),
    0
};

/**/
int
setup_(UNUSED(Module m))
{
    return 0;
}

/**/
int
features_(Module m, char ***features)
{
    *features = featuresarray(m, &module_features);
    return 0;
}

/**/
int
enables_(Module m, int **enables)
{
    return handlefeatures(m, &module_features, enables);
}

/**/
int
boot_(UNUSED(Module m))
{
    zgdbm_tied = zshcalloc((1) * sizeof(char *));
    return 0;
}

/**/
int
cleanup_(Module m)
{
    /* This frees `zgdbm_tied` */
    return setfeatureenables(m, &module_features, NULL);
}

/**/
int
finish_(UNUSED(Module m))
{
    return 0;
}

/*********************
 * Utility functions *
 *********************/

static Param createhash( char *name, int flags ) {
    Param pm;
    HashTable ht;

    pm = createparam(name, PM_SPECIAL | PM_HASHED);
    if (!pm) {
        return NULL;
    }

    if (pm->old)
	pm->level = locallevel;

    /* This creates standard hash. */
    ht = pm->u.hash = newparamtable(32, name);
    if (!pm->u.hash) {
        paramtab->removenode(paramtab, name);
        paramtab->freenode(&pm->node);
        zwarnnam(name, "Out of memory when allocating hash");
    }

    /* These provide special features */
    ht->getnode = ht->getnode2 = getgdbmnode;
    ht->scantab = scangdbmkeys;

    return pm;
}

/*
 * Adds parameter name to `zgdbm_tied`
 */

static int append_tied_name( const char *name ) {
    int old_len = arrlen(zgdbm_tied);
    char **new_zgdbm_tied = zshcalloc( (old_len+2) * sizeof(char *));

    /* Copy */
    char **p = zgdbm_tied;
    char **dst = new_zgdbm_tied;
    while (*p) {
        *dst++ = *p++;
    }

    /* Append new one */
    *dst = ztrdup(name);

    /* Substitute, free old one */
    zfree(zgdbm_tied, sizeof(char *) * (old_len + 1));
    zgdbm_tied = new_zgdbm_tied;

    return 0;
}

/*
 * Removes parameter name from `zgdbm_tied`
 */

static int remove_tied_name( const char *name ) {
    int old_len = arrlen(zgdbm_tied);

    /* Two stage, to always have arrlen() == zfree-size - 1.
     * Could do allocation and revert when `not found`, but
     * what would be better about that. */

    /* Find one to remove */
    char **p = zgdbm_tied;
    while (*p) {
        if (0==strcmp(name,*p)) {
            break;
        }
        p++;
    }

    /* Copy x+1 to x */
    while (*p) {
        *p=*(p+1);
        p++;
    }

    /* Second stage. Size changed? Only old_size-1
     * change is possible, but.. paranoia way */
    int new_len = arrlen(zgdbm_tied);
    if (new_len != old_len) {
        char **new_zgdbm_tied = zshcalloc((new_len+1) * sizeof(char *));

        /* Copy */
        p = zgdbm_tied;
        char **dst = new_zgdbm_tied;
        while (*p) {
            *dst++ = *p++;
        }
        *dst = NULL;

        /* Substitute, free old one */
        zfree(zgdbm_tied, sizeof(char *) * (old_len + 1));
        zgdbm_tied = new_zgdbm_tied;
    }

    return 0;
}

/*
 * Unmetafy that:
 * - duplicates bufer to work on it,
 * - does zalloc of exact size for the new string,
 * - restores work buffer to original content, to restore strlen
 *
 * No zsfree()-confusing string will be produced.
 */
char *unmetafy_zalloc(const char *to_copy, int *new_len) {
    char *work, *to_return;
    int my_new_len = 0;

    work = ztrdup(to_copy);
    work = unmetafy(work,&my_new_len);

    if (new_len)
        *new_len = my_new_len;

    /* This string can be correctly zsfree()-d */
    to_return = (char *) zalloc((my_new_len+1)*sizeof(char));
    memcpy(to_return, work, sizeof(char)*my_new_len); // memcpy handles $'\0'
    to_return[my_new_len]='\0';

    /* Restore original strlen and correctly free */
    strcpy(work, to_copy);
    zsfree(work);

    return to_return;
}


Messages sorted by: Reverse Date, Date, Thread, Author