Index: trunk/doc/devel/TODO
===================================================================
--- trunk/doc/devel/TODO	(revision 261)
+++ trunk/doc/devel/TODO	(revision 262)
@@ -15,4 +15,12 @@
    descriptor information.  Maybe by MTIME as well.
 
+ - reglookup-timeline needs to be replaced with something cross-platform.  
+   Perhaps a python script that provides MTIME range filtering capabilities.
+
+ - Need to integrate much of reglookup-recover's algorithms into regfi 
+   and then expose them from the bottom-up to provide building blocks 
+   through regfi and pyregfi.  This should be addressed along with code 
+   to support handling of partial/fragmented registry hives.
+
  - Testing, testing, and more testing.  reglookup needs to be more
    heavily tested on all recent Windows platforms.  A regression test
@@ -26,11 +34,12 @@
    to be decent, UTF-8 output would be nice.
 
- - Develop and solidify regfi API.  Regfi should be better documented and 
-   eventually needs a set of higher-language wrappers, starting with Python
-   and possibly moving on to Perl as well.
+ - Continue to improve regfi/pyregfi APIs as needed.  winsec library needs more
+   flexibility and documentation.
+
+ - Consider adding regfi wrappers for other high-level languages (perl? ruby?).
 
  - Documentation.  The security descriptor output format needs to be
    documented.  Also, function contracts should be added to the
-   lower-level functions of regfi.c.
+   lower-level functions of regfi.c. Continue adding
 
  - Consider switching from libiconv to Joachim Metz's libuna for
@@ -39,4 +48,7 @@
  - Grep through the source for 'XXX', and you'll find more.
 
+ - Consider integrating packaging rules for debian/other platforms into trunk.
+
+ - Investigate why file descriptors can't be directly used in Windows
 
 
@@ -44,19 +56,7 @@
 ===========
 
-Add fields/methods for accessing security descriptors in pyregfi
-
-convert MTIME structure to uint64_t if possible
-
-investigate why file descriptors can't be directly used in Windows
-
-Fill in and update remaining regfi/pyregfi API documentation
-
-Possible debian package build rules
-
-Possibly replace reglookup-timeline with something cross-platform
-
 Testing
   Full diffs
   regfi and pyregfi threading
   valgrind in multiple scenarios for reglookup, reglookup-recover
-
+  double check man pages
Index: trunk/include/regfi.h
===================================================================
--- trunk/include/regfi.h	(revision 261)
+++ trunk/include/regfi.h	(revision 262)
@@ -1,4 +1,4 @@
 /*
- * Copyright (C) 2005-2010 Timothy D. Morgan
+ * Copyright (C) 2005-2011 Timothy D. Morgan
  * Copyright (C) 2010 Michael Cohen
  * Copyright (C) 2005 Gerald (Jerry) Carter
@@ -1152,5 +1152,5 @@
  * @param file  the file from which key is derived
  * @param key   the key whose subkey is desired
- * @param name  name of the desired subkey
+ * @param name  name of the desired subkey (case-insensitive)
  * @param index a return value: the index of the desired subkey.
  *              undefined on error
@@ -1171,5 +1171,5 @@
  * @param file  the file from which key is derived
  * @param key   the key whose value is desired
- * @param name  name of the desired value
+ * @param name  name of the desired value (case-insensitive)
  * @param index a return value: the index of the desired value.  
  *              undefined on error
@@ -1753,8 +1753,6 @@
 _EXPORT()
 int32_t               regfi_calc_maxsize(REGFI_FILE* file, uint32_t offset);
-int32_t               regfi_conv_charset(const char* input_charset, 
-					 const char* output_charset,
-					 uint8_t* input, char* output, 
-					 uint32_t input_len, uint32_t output_max);
+REGFI_BUFFER          regfi_conv_charset(const char* input_charset, const char* output_charset,
+                                         uint8_t* input, uint32_t input_len);
 _EXPORT()
 REGFI_DATA*           regfi_buffer_to_data(REGFI_BUFFER raw_data);
Index: trunk/lib/regfi.c
===================================================================
--- trunk/lib/regfi.c	(revision 261)
+++ trunk/lib/regfi.c	(revision 262)
@@ -1186,5 +1186,5 @@
    *      when recovering deleted VK records.
    */
-  int32_t tmp_size;
+  REGFI_BUFFER tmp_buf;
   REGFI_ENCODING from_encoding = (vk->flags & REGFI_VK_FLAG_ASCIINAME)
     ? REGFI_ENCODING_ASCII : REGFI_ENCODING_UTF16LE;
@@ -1200,20 +1200,19 @@
   else
   {
-    vk->name = talloc_array(vk, char, vk->name_length+1);
-    if(vk->name == NULL)
-      return;
-
-    tmp_size = regfi_conv_charset(regfi_encoding_int2str(from_encoding),
-				  regfi_encoding_int2str(output_encoding),
-				  vk->name_raw, vk->name,
-				  vk->name_length, vk->name_length+1);
-    if(tmp_size < 0)
+    tmp_buf = regfi_conv_charset(regfi_encoding_int2str(from_encoding),
+                                 regfi_encoding_int2str(output_encoding),
+                                 vk->name_raw, vk->name_length);
+    if(tmp_buf.buf == NULL)
     {
       regfi_log_add(REGFI_LOG_WARN, "Error occurred while converting"
 			" value name to encoding %s.  Error message: %s",
 			regfi_encoding_int2str(output_encoding), 
-			strerror(-tmp_size));
-      talloc_free(vk->name);
+			strerror(errno));
       vk->name = NULL;
+    }
+    else
+    {
+      vk->name = (char*)tmp_buf.buf;
+      talloc_reparent(NULL, vk, vk->name);
     }
   }
@@ -1280,5 +1279,5 @@
    *      when recovering deleted NK records.
    */
-  int32_t tmp_size;
+  REGFI_BUFFER tmp_buf;
   REGFI_ENCODING from_encoding = (nk->flags & REGFI_NK_FLAG_ASCIINAME) 
     ? REGFI_ENCODING_ASCII : REGFI_ENCODING_UTF16LE;
@@ -1294,22 +1293,19 @@
   else
   {
-    nk->name = talloc_array(nk, char, nk->name_length+1);
-    if(nk->name == NULL)
-      return;
-
-    memset(nk->name,0,nk->name_length+1);
-
-    tmp_size = regfi_conv_charset(regfi_encoding_int2str(from_encoding),
-				  regfi_encoding_int2str(output_encoding),
-				  nk->name_raw, nk->name,
-				  nk->name_length, nk->name_length+1);
-    if(tmp_size < 0)
+    tmp_buf = regfi_conv_charset(regfi_encoding_int2str(from_encoding),
+                                 regfi_encoding_int2str(output_encoding),
+                                 nk->name_raw, nk->name_length);
+    if(tmp_buf.buf == NULL)
     {
       regfi_log_add(REGFI_LOG_WARN, "Error occurred while converting"
-			" key name to encoding %s.  Error message: %s",
-			regfi_encoding_int2str(output_encoding), 
-			strerror(-tmp_size));
-      talloc_free(nk->name);
+                    " key name to encoding %s.  Error message: %s",
+                    regfi_encoding_int2str(output_encoding), 
+                    strerror(errno));
       nk->name = NULL;
+    }
+    else
+    {
+      nk->name = (char*)tmp_buf.buf;
+      talloc_reparent(NULL, nk, nk->name);
     }
   }
@@ -2243,7 +2239,7 @@
   REGFI_CLASSNAME* ret_val;
   uint8_t* raw;
-  char* interpreted;
+  REGFI_BUFFER tmp_buf;
   uint32_t offset;
-  int32_t conv_size, max_size;
+  int32_t max_size;
   uint16_t parse_length;
 
@@ -2276,24 +2272,18 @@
   talloc_reparent(NULL, ret_val, raw);
 
-  interpreted = talloc_array(NULL, char, parse_length);
-
-  conv_size = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
-				 regfi_encoding_int2str(file->string_encoding),
-				 raw, interpreted,
-				 parse_length, parse_length);
-  if(conv_size < 0)
+  tmp_buf = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
+                               regfi_encoding_int2str(file->string_encoding),
+                               raw, parse_length);
+  if(tmp_buf.buf == NULL)
   {
     regfi_log_add(REGFI_LOG_WARN, "Error occurred while"
 		  " converting classname to charset %s.  Error message: %s",
-		  file->string_encoding, strerror(-conv_size));
-    talloc_free(interpreted);
+		  file->string_encoding, strerror(errno));
     ret_val->interpreted = NULL;
   }
   else
   {
-    /* XXX: check for NULL return here? */
-    interpreted = talloc_realloc(NULL, interpreted, char, conv_size);
-    ret_val->interpreted = interpreted;
-    talloc_reparent(NULL, ret_val, interpreted);
+    ret_val->interpreted = (char*)tmp_buf.buf;
+    talloc_reparent(NULL, ret_val, tmp_buf.buf);
   }
 
@@ -2365,4 +2355,8 @@
     return false;
 
+  /* XXX: Should lazily build a hash table in memory to index where keys are when
+   *      there are a large number of subkeys.  Attach this to cached keys to
+   *      bound the extra amount of memory used. 
+   */
   for(i=0; (i < num_subkeys) && (found == false); i++)
   {
@@ -2396,4 +2390,8 @@
   bool found = false;
 
+  /* XXX: Should lazily build a hash table in memory to index where values are when
+   *      there are a large number of them.  Attach this to cached keys to
+   *      bound the extra amount of memory used. 
+   */
   for(i=0; (i < num_values) && (found == false); i++)
   {
@@ -2494,8 +2492,7 @@
 			  uint32_t type, REGFI_DATA* data)
 {
+  REGFI_BUFFER tmp_buf;
   uint8_t** tmp_array;
-  uint8_t* tmp_str;
-  int32_t tmp_size;
-  uint32_t i, j, array_size;
+  uint32_t i, j;
 
   if(data == NULL)
@@ -2508,33 +2505,21 @@
   /* REG_LINK is a symbolic link, stored as a unicode string. */
   case REG_LINK:
-    tmp_str = talloc_array(NULL, uint8_t, data->size);
-    if(tmp_str == NULL)
-    {
+    tmp_buf = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
+                                 regfi_encoding_int2str(string_encoding),
+                                 data->raw, data->size);
+    if(tmp_buf.buf == NULL)
+    {
+      regfi_log_add(REGFI_LOG_INFO, "Error occurred while"
+		    " converting data of type %d to string encoding %d."
+                    "  Error message: %s",
+		    type, string_encoding, strerror(errno));
       data->interpreted.string = NULL;
       data->interpreted_size = 0;
       return false;
     }
-      
-    tmp_size = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
-				  regfi_encoding_int2str(string_encoding),
-				  data->raw, (char*)tmp_str, 
-				  data->size, data->size);
-    if(tmp_size < 0)
-    {
-      regfi_log_add(REGFI_LOG_INFO, "Error occurred while"
-		    " converting data of type %d to %d.  Error message: %s",
-		    type, string_encoding, strerror(-tmp_size));
-      talloc_free(tmp_str);
-      data->interpreted.string = NULL;
-      data->interpreted_size = 0;
-      return false;
-    }
-
-    tmp_str = talloc_realloc(NULL, tmp_str, uint8_t, tmp_size);
-    if(tmp_str == NULL)
-      return false;
-    data->interpreted.string = tmp_str;
-    data->interpreted_size = tmp_size;
-    talloc_reparent(NULL, data, tmp_str);
+
+    data->interpreted.string = tmp_buf.buf;
+    data->interpreted_size = tmp_buf.len;
+    talloc_reparent(NULL, data, tmp_buf.buf);
     break;
 
@@ -2574,7 +2559,16 @@
     
   case REG_MULTI_SZ:
-    tmp_str = talloc_array(NULL, uint8_t, data->size);
-    if(tmp_str == NULL)
-    {
+    /* Attempt to convert entire string from UTF-16LE to output encoding,
+     * then parse and quote fields individually.
+     */
+    tmp_buf = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
+                                 regfi_encoding_int2str(string_encoding),
+                                 data->raw, data->size);
+    if(tmp_buf.buf == NULL)
+    {
+      regfi_log_add(REGFI_LOG_INFO, "Error occurred while"
+		    " converting data of type %d to string encoding %d."
+                    "  Error message: %s",
+		    type, string_encoding, strerror(errno));
       data->interpreted.multiple_string = NULL;
       data->interpreted_size = 0;
@@ -2582,37 +2576,19 @@
     }
 
-    /* Attempt to convert entire string from UTF-16LE to output encoding,
-     * then parse and quote fields individually.
-     */
-    tmp_size = regfi_conv_charset(regfi_encoding_int2str(REGFI_ENCODING_UTF16LE),
-				  regfi_encoding_int2str(string_encoding),
-				  data->raw, (char*)tmp_str,
-				  data->size, data->size);
-    if(tmp_size < 0)
-    {
-      regfi_log_add(REGFI_LOG_INFO, "Error occurred while"
-		    " converting data of type %d to %s.  Error message: %s",
-		    type, string_encoding, strerror(-tmp_size));
-      talloc_free(tmp_str);
-      data->interpreted.multiple_string = NULL;
-      data->interpreted_size = 0;
-      return false;
-    }
-
-    array_size = tmp_size+1;
-    tmp_array = talloc_array(NULL, uint8_t*, array_size);
+    tmp_array = talloc_array(NULL, uint8_t*, tmp_buf.len+1);
     if(tmp_array == NULL)
     {
-      talloc_free(tmp_str);
+      talloc_free(tmp_buf.buf);
       data->interpreted.string = NULL;
       data->interpreted_size = 0;
       return false;
     }
-    
-    tmp_array[0] = tmp_str;
-    for(i=0,j=1; i < tmp_size && j < array_size-1; i++)
-    {
-      if(tmp_str[i] == '\0' && (i+1 < tmp_size) && tmp_str[i+1] != '\0')
-	tmp_array[j++] = tmp_str+i+1;
+
+    tmp_array[0] = tmp_buf.buf;
+    for(i=0,j=1; i < tmp_buf.len && j < tmp_buf.len; i++)
+    {
+      if(tmp_buf.buf[i] == '\0' && (i+1 < tmp_buf.len) 
+         && tmp_buf.buf[i+1] != '\0')
+	tmp_array[j++] = tmp_buf.buf+i+1;
     }
     tmp_array[j] = NULL;
@@ -2620,6 +2596,6 @@
     data->interpreted.multiple_string = tmp_array;
     /* XXX: how meaningful is this?  should we store number of strings instead? */
-    data->interpreted_size = tmp_size;
-    talloc_reparent(NULL, tmp_array, tmp_str);
+    data->interpreted_size = tmp_buf.len;
+    talloc_reparent(NULL, tmp_array, tmp_buf.buf);
     talloc_reparent(NULL, data, tmp_array);
     break;
@@ -2663,38 +2639,80 @@
 
 /******************************************************************************
- * Convert from UTF-16LE to specified character set. 
- * On error, returns a negative errno code.
+ * Convert string from input_charset to output_charset.
+ * On error, returns a NULL buf attribute and sets the errno.
  *****************************************************************************/
-int32_t regfi_conv_charset(const char* input_charset, const char* output_charset,
-			   uint8_t* input, char* output, 
-			   uint32_t input_len, uint32_t output_max)
+REGFI_BUFFER regfi_conv_charset(const char* input_charset, const char* output_charset,
+                                uint8_t* input, uint32_t input_len)
 {
   iconv_t conv_desc;
   char* inbuf = (char*)input;
-  char* outbuf = output;
-  size_t in_len = (size_t)input_len;
-  size_t out_len = (size_t)(output_max-1);
+  char* outbuf;
+  char* retbuf;
+  size_t allocated = (size_t)input_len;
+  size_t in_left = (size_t)input_len;
+  size_t out_left = (size_t)allocated-1;
+  REGFI_BUFFER ret_val;
   int ret;
 
+  ret_val.buf = NULL;
+  ret_val.len = 0;
+  retbuf = talloc_array(NULL, char, allocated);
+  outbuf = retbuf;
+  if(outbuf == NULL)
+  {
+    errno = ENOMEM;
+    return ret_val;
+  }
+
+  /* Set up conversion descriptor. */
   /* XXX: Consider creating a couple of conversion descriptors earlier,
    *      storing them on an iterator so they don't have to be recreated
    *      each time.
    */
-
-  /* Set up conversion descriptor. */
   conv_desc = iconv_open(output_charset, input_charset);
 
-  ret = iconv(conv_desc, &inbuf, &in_len, &outbuf, &out_len);
+  ret = 0;
+  do
+  {
+    if(ret == -1)
+    {
+      retbuf = talloc_realloc(NULL, retbuf, char, allocated+(in_left*2));
+      if(retbuf == NULL)
+      {
+        errno = ENOMEM;
+        return ret_val;
+      }
+      outbuf = retbuf+(allocated-1-out_left);
+      out_left += in_left*2;
+      allocated += in_left*2;
+    }
+    ret = iconv(conv_desc, &inbuf, &in_left, &outbuf, &out_left);
+    
+  } while(ret == -1 && errno == E2BIG);
+  
   if(ret == -1)
   {
     iconv_close(conv_desc);
-    return -errno;
-  }
-  *outbuf = '\0';
-
-  iconv_close(conv_desc);  
-  return output_max-out_len-1;
-}
-
+    return ret_val;
+  }
+
+  /* Save memory */
+  if(out_left > 0)
+  {
+    retbuf = talloc_realloc(NULL, retbuf, char, allocated-out_left);
+    if(retbuf == NULL)
+    {
+      errno = ENOMEM;
+      return ret_val;
+    }
+    allocated -= out_left;
+  }
+  retbuf[allocated-1] = '\0';
+  iconv_close(conv_desc);
+
+  ret_val.buf = (uint8_t*)retbuf;
+  ret_val.len = allocated-1;
+  return ret_val;
+}
 
 
Index: trunk/python/pyregfi/__init__.py
===================================================================
--- trunk/python/pyregfi/__init__.py	(revision 261)
+++ trunk/python/pyregfi/__init__.py	(revision 262)
@@ -106,6 +106,4 @@
 # @note Developers strive to make pyregfi thread-safe.
 # 
-# @note Key and Value names are case-sensitive in regfi and pyregfi
-#
 import sys
 import time
@@ -390,13 +388,15 @@
     #
     # @param name The name of the subkey or value desired.  
-    #             This is case-sensitive.
-    #
-    # @note The registry format does inherently prevent multiple
-    #       subkeys or values from having the same name.  
-    #       This interface simply returns the first match.  
+    #             This is case-insensitive.
+    #
+    # @note The registry format does not inherently prevent multiple
+    #       subkeys or values from having the same name, having a key
+    #       and a value with the same name, or having the same name in
+    #       different cases that could both match. 
+    #       This interface simply returns the first match in the list.
     #       Lookups using this method could also fail due to incorrectly
-    #       encoded strings.
-    #       To identify any duplicates, use the iterator interface to 
-    #       check every list element.
+    #       encoded strings stored as names.
+    #       To identify any duplicates or elements with malformed names,
+    #       use the iterator interface to check every list element.
     #
     # @return the first element whose name matches, or None if the element
