diff -r 1edf5cd93bb9 src/prevalence.lisp
--- a/src/prevalence.lisp	Sun Jul 26 11:10:13 2009 +0200
+++ b/src/prevalence.lisp	Sat Sep 26 15:22:42 2009 +0400
@@ -103,7 +103,11 @@
    (transaction-hook ;; type function
                      :accessor get-transaction-hook
                      :initarg :transaction-hook
-                     :initform #'identity))
+                     :initform #'identity)
+   (external-format ;; external encoding for file operations
+                    :reader get-external-format
+                    :initarg :external-format
+                    :initform :utf-8))
   (:documentation "Base Prevalence system implementation object"))
 
 (defclass guarded-prevalence-system (prevalence-system)
@@ -156,7 +160,8 @@
       (setf transaction-log-stream (open (get-transaction-log system)
 					 :direction :output
 					 :if-does-not-exist :create
-					 :if-exists :append)))))
+					 :if-exists :append
+                                         :external-format (get-external-format system))))))
 
 (defmethod close-open-streams ((system prevalence-system) &key abort)
   "Close all open stream associated with system (optionally aborting operations in progress)"
@@ -238,17 +243,17 @@
 	(snapshot (get-snapshot system)))
     (close-open-streams system)
     (when (probe-file snapshot)
-      (copy-file snapshot (merge-pathnames (make-pathname :name (get-snapshot-filename system timetag)
-                                                          :type (get-file-extension system))
-                                           snapshot)))
+      (rename-file snapshot (merge-pathnames (make-pathname :name (get-snapshot-filename system timetag)
+                                                            :type (get-file-extension system))
+                                             snapshot)))
     (with-open-file (out snapshot
-			 :direction :output :if-does-not-exist :create :if-exists :supersede)
+			 :direction :output :if-does-not-exist :create :if-exists :supersede :external-format (get-external-format system))
+      (s-serialization::reset (get-serialization-state system))
       (funcall (get-serializer system) (get-root-objects system) out (get-serialization-state system)))
     (when (probe-file transaction-log)
-      (copy-file transaction-log (merge-pathnames (make-pathname :name (get-transaction-log-filename system timetag)
-                                                                 :type (get-file-extension system))
-                                                  transaction-log))
-      (delete-file transaction-log))))
+      (rename-file transaction-log (merge-pathnames (make-pathname :name (get-transaction-log-filename system timetag)
+                                                                   :type (get-file-extension system))
+                                                    transaction-log)))))
 
 (defmethod backup ((system prevalence-system) &key directory)
   "Make backup copies of the current snapshot and transaction-log files"
@@ -273,7 +278,7 @@
   (clrhash (get-root-objects system))
   (close-open-streams system)
   (when (probe-file (get-snapshot system))
-    (with-open-file (in (get-snapshot system) :direction :input)
+    (with-open-file (in (get-snapshot system) :direction :input :external-format (get-external-format system))
       (setf (get-root-objects system) (funcall (get-deserializer system) in (get-serialization-state system)))))
   (when (probe-file (get-transaction-log system))
     (let ((position 0))
@@ -284,7 +289,7 @@
                                   condition)
                           (truncate-file (get-transaction-log system) position)
                           (return-from restore))))
-	(with-open-file (in (get-transaction-log system) :direction :input)
+	(with-open-file (in (get-transaction-log system) :direction :input :external-format (get-external-format system))
 	  (loop
 	   (let ((transaction (funcall (get-deserializer system) in (get-serialization-state system))))
 	     (setf position (file-position in))
@@ -310,7 +315,7 @@
 (defmethod backup ((system guarded-prevalence-system) &key directory)
   "Do a backup on a system controlled by a guard"
   (funcall (get-guard system)
-           #'(lambda () (call-next-method system directory))))
+           #'(lambda () (call-next-method system :directory directory))))
 
 (defmethod restore ((system guarded-prevalence-system))
   "Restore a system controlled by a guard"
diff -r 1edf5cd93bb9 src/serialization/sexp.lisp
--- a/src/serialization/sexp.lisp	Sun Jul 26 11:10:13 2009 +0200
+++ b/src/serialization/sexp.lisp	Sat Sep 26 15:22:42 2009 +0400
@@ -3,12 +3,10 @@
 
 (defun serialize-sexp (object stream &optional (serialization-state (make-serialization-state)))
   "Write a serialized version of object to stream using s-expressions, optionally reusing a serialization-state"
-  (reset serialization-state)
   (serialize-sexp-internal object stream serialization-state))
 
 (defun deserialize-sexp (stream &optional (serialization-state (make-serialization-state)))
   "Read and return an s-expression serialized version of a lisp object from stream, optionally reusing a serialization state"
-  (reset serialization-state)
   (let ((sexp (read stream nil :eof)))
     (if (eq sexp :eof) 
         nil
@@ -145,8 +143,9 @@
                     (write-string " . " stream)
                     (serialize-sexp-internal (slot-value object slot) stream serialization-state)
                     (write-string ")" stream))
-                serializable-slots))
-	(write-string " ) )" stream)))))
+                serializable-slots)
+          (write-string " )" stream))
+	(write-string " )" stream)))))
 
 ;;; objects
 (defmethod serialize-sexp-internal ((object standard-object) stream serialization-state)
diff -r 1edf5cd93bb9 src/serialization/xml.lisp
--- a/src/serialization/xml.lisp	Sun Jul 26 11:10:13 2009 +0200
+++ b/src/serialization/xml.lisp	Sat Sep 26 15:22:42 2009 +0400
@@ -3,12 +3,10 @@
 
 (defun serialize-xml (object stream &optional (serialization-state (make-serialization-state)))
   "Write a serialized version of object to stream using XML, optionally reusing a serialization-state"
-  (reset serialization-state)
   (serialize-xml-internal object stream serialization-state))
 
 (defun deserialize-xml (stream &optional (serialization-state (make-serialization-state)))
   "Read and return an XML serialized version of a lisp object from stream, optionally reusing a serialization state"
-  (reset serialization-state)
   (let ((*deserialized-objects* (get-hashtable serialization-state)))
     (declare (special *deserialized-objects*))
     (car (s-xml:start-parse-xml stream (get-xml-parser-state serialization-state)))))
diff -r 1edf5cd93bb9 test/cl-prevalence-test.asd
--- a/test/cl-prevalence-test.asd	Sun Jul 26 11:10:13 2009 +0200
+++ b/test/cl-prevalence-test.asd	Sat Sep 26 15:22:42 2009 +0400
@@ -14,5 +14,7 @@
      (:file "test-managed-prevalence")
      (:file "test-master-slave")
      (:file "test-serialization")
+     (:file "test-external-format")
+     (:file "test-complex-serialize")
      )
     :depends-on (:cl-prevalence :fiveam))
\ No newline at end of file
diff -r 1edf5cd93bb9 test/test-complex-serialize.lisp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/test-complex-serialize.lisp	Sat Sep 26 15:22:42 2009 +0400
@@ -0,0 +1,233 @@
+
+(in-package :cl-prevalence-test)
+
+(def-suite test-complex-serialize :in cl-prevalence-test)
+
+(in-suite test-complex-serialize)
+
+(defparameter *test-system-directory* (pathname "/tmp/test-complex-serialize/"))
+
+(defun make-specific-prevalence-system (type)
+  (let ((directory *test-system-directory*))
+    (make-instance 'prevalence-system
+                   :directory directory
+                   :serializer (ecase type
+                                 (:xml #'s-serialization:serialize-xml)
+                                 (:sexp #'s-serialization:serialize-sexp))
+                   :deserializer (ecase type
+                                   (:xml #'s-serialization:deserialize-xml)
+                                   (:sexp #'s-serialization:deserialize-sexp))
+                   :file-extension (ecase type (:xml "xml") (:sexp "lisp"))
+                   :external-format :utf-8)))
+
+(defvar *chars* (coerce (loop
+                           :for i :from (char-code #\a) :to (char-code #\z)
+                           :for c = (code-char i)
+                           :collect c 
+                           :collect (char-upcase c))
+                        'string))
+
+(defun gen-random-name (&optional (len 32))
+  (coerce (loop
+             :repeat len
+             :collect (elt *chars* (random (length *chars*))))
+          'string))
+
+(defclass item ()
+  ((unique-id :initarg :unique-id
+              :reader item-unique-id)
+   (shared-id :initarg :shared-id
+              :reader item-shared-id)
+   (name :initarg :name
+         :reader item-name)
+   (container :initarg :container
+              :reader item-container)))
+
+(defclass items-box ()
+  ((box-name :initarg :box-name
+             :reader items-box-name)
+   (items :initform '()
+          :accessor items-box-items)
+   (unique-db :initform (make-hash-table)
+              :reader items-box-unique-db)
+   (shared-db :initform (make-hash-table)
+              :reader items-box-shared-db)
+   (names-idx :initform (make-hash-table :test 'equal)
+              :reader items-box-names-idx)))
+
+(defgeneric create-item (box unique-id shared-id name container))
+
+(defmethod create-item ((box items-box) unique-id shared-id name (container items-box))
+  (let* ((item (make-instance 'item
+                              :unique-id unique-id
+                              :shared-id shared-id
+                              :name name
+                              :container container)))
+    (push item (items-box-items box))
+    (setf (gethash unique-id (items-box-unique-db box)) item)
+    (push item (gethash shared-id (items-box-shared-db box)))
+    (setf (gethash name (items-box-names-idx box)) item)
+    item))
+
+(defun create-items-box (name)
+  (make-instance 'items-box :box-name name))
+
+
+
+
+(defvar *boxes* nil)
+
+(defun tx-create-box (system box)
+  (let ((handler (or (get-root-object system :handler)
+                     (setf (get-root-object system :handler)
+                           (make-hash-table :test 'equal)))))
+    (setf (gethash (items-box-name box) handler) box)))
+
+(defun tx-create-item (system box unique-id shared-id name container)
+  (declare (ignore system))
+  (create-item box unique-id shared-id name container))
+
+
+(defun gen-boxes (system)
+  (loop :repeat 5 :collect (execute-transaction (tx-create-box system (create-items-box (gen-random-name))))))
+
+(defun gen-items-for-box (system box)
+  (loop
+     :repeat 2000
+     :for unique-id = (loop
+                         :for id = (random 1000000)
+                         :while (gethash id (items-box-unique-db box))
+                         :finally (return id))
+     :do (execute-transaction (tx-create-item system box unique-id (random 100) (gen-random-name) box))))
+
+
+(defun check-cache-against-system (system)
+  (loop
+     :for name :being :the :hash-keys :in *boxes*
+     :do (unless (boxes-equal? (gethash name *boxes*)
+                               (gethash name (get-root-object system :handler)))
+           (return nil))
+     :finally (return t)))
+
+(defun items-lists-equal? (list-a list-b)
+  (loop
+     :for item-a :in list-a
+     :for item-b :in list-b
+     :do (unless (items-equal? item-a item-b)
+           (return nil))
+     :finally (return t)))
+
+(defun indexes-equal? (checker idx-a idx-b)
+  (and (= (hash-table-count idx-a) (hash-table-count idx-a))
+       (loop
+          :for key :being :the :hash-keys :in idx-a
+          :do (unless (funcall checker
+                               (gethash key idx-a)
+                               (gethash key idx-b))
+                (return nil))
+          :finally (return t))))
+
+(defun items-equal? (item-a item-b)
+  (and (= (item-unique-id item-a) (item-unique-id item-b))
+       (= (item-shared-id item-a) (item-shared-id item-b))
+       (equal (item-name item-a) (item-name item-b))
+       (equal (items-box-name (item-container item-a))
+              (items-box-name (item-container item-b)))))
+
+(defun boxes-equal? (box-a box-b)
+  (and (equal (items-box-name box-a)
+              (items-box-name box-b))
+       (= (length (items-box-items box-a))
+          (length (items-box-items box-b)))
+       (items-lists-equal? (items-box-items box-a)
+                           (items-box-items box-b))
+       (indexes-equal? #'items-equal?
+                       (items-box-unique-db box-a)
+                       (items-box-unique-db box-b))
+       (indexes-equal? #'items-lists-equal?
+                       (items-box-shared-db box-a)
+                       (items-box-shared-db box-b))
+       (indexes-equal? #'items-equal?
+                       (items-box-names-idx box-a)
+                       (items-box-names-idx box-b))))
+
+
+(defun regenerate-data ()
+  (setf *boxes* (make-hash-table :test 'equal))
+  (let ((boxes (gen-boxes *test-system*)))
+    (mapc (lambda (box)
+            (setf (gethash (items-box-name box) *boxes*) box)
+            (gen-items-for-box *test-system* box))
+          boxes)))
+  
+(defvar *test-system* nil)
+
+(test test-complex-create-system-xml
+  (setf *test-system* nil)
+  (let ((directory *test-system-directory*))
+    ;; Throw away any log files that we find: we want to start from scratch
+    (when (probe-file directory)
+      (dolist (pathname (directory (merge-pathnames "*.xml" directory)))
+        (delete-file pathname)))
+    (setf *test-system* (make-specific-prevalence-system :xml))
+    (is-true *test-system*)))
+
+(test test-complex-fill-xml
+  (regenerate-data)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-restore-xml
+  (setf *test-system* (make-specific-prevalence-system :xml))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-snapshot/restore-xml
+  (snapshot *test-system*)
+  (setf *test-system* (make-specific-prevalence-system :xml))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-more-after-snapshot-xml
+  (regenerate-data)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-more-after-snapshot-restore-xml
+  (setf *test-system* (make-specific-prevalence-system :xml))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+    
+
+(test test-complex-create-system-sexp
+  (setf *test-system* nil)
+  (let ((directory *test-system-directory*))
+    ;; Throw away any log files that we find: we want to start from scratch
+    (when (probe-file directory)
+      (dolist (pathname (directory (merge-pathnames "*.lisp" directory)))
+        (delete-file pathname)))
+    (setf *test-system* (make-specific-prevalence-system :sexp))
+    (is-true *test-system*)))
+
+(test test-complex-fill-sexp
+  (regenerate-data)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-restore-sexp
+  (setf *test-system* (make-specific-prevalence-system :sexp))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-snapshot/restore-sexp
+  (snapshot *test-system*)
+  (setf *test-system* (make-specific-prevalence-system :sexp))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-more-after-snapshot-sexp
+  (regenerate-data)
+  (is (check-cache-against-system *test-system*)))
+
+(test test-complex-more-after-snapshot-restore-sexp
+  (setf *test-system* (make-specific-prevalence-system :sexp))
+  (is-true *test-system*)
+  (is (check-cache-against-system *test-system*)))
+
diff -r 1edf5cd93bb9 test/test-external-format.lisp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/test-external-format.lisp	Sat Sep 26 15:22:42 2009 +0400
@@ -0,0 +1,70 @@
+
+(in-package :cl-prevalence-test)
+
+(def-suite test-external-format :in cl-prevalence-test)
+
+(in-suite test-external-format)
+
+(defparameter *test-system-directory* (pathname "/tmp/test-external-format/"))
+
+(defun make-prevalence-system-w/format (external-format)
+  (let ((directory *test-system-directory*))
+    (make-instance 'prevalence-system
+                   :directory directory
+                   :serializer #'s-serialization:serialize-sexp
+                   :deserializer #'s-serialization:deserialize-sexp
+                   :file-extension "lisp"
+                   :external-format external-format)))
+
+(defvar *test-system* nil)
+
+(test test-start/utf-8
+ "Create a new clean prevalence system for testing purposes"
+  (let ((directory *test-system-directory*))
+    ;; Throw away any .lisp files that we find: we want to start from scratch
+    (when (probe-file directory)
+      (dolist (pathname (directory (merge-pathnames "*.lisp" directory)))
+        (delete-file pathname)))
+    (setf *test-system* (make-prevalence-system-w/format :utf-8))
+    (is-true *test-system*)))
+
+(defvar *test-string/utf-8* (coerce '(#\CYRILLIC_CAPITAL_LETTER_E
+                                      #\CYRILLIC_SMALL_LETTER_TE
+                                      #\CYRILLIC_SMALL_LETTER_O
+                                      #\Space
+                                      #\CYRILLIC_SMALL_LETTER_TE
+                                      #\CYRILLIC_SMALL_LETTER_IE
+                                      #\CYRILLIC_SMALL_LETTER_ES
+                                      #\CYRILLIC_SMALL_LETTER_TE)
+                                    'string))
+
+(defun tx-set-string (store string)
+  (setf (get-root-object store :string) string))
+
+(defun get-string ()
+  (get-root-object *test-system* :string))
+
+(test test-save/utf-8
+  (is (null (get-string)))
+  (execute-transaction (tx-set-string *test-system* *test-string/utf-8*))
+  (is (equal (get-string) *test-string/utf-8*)))
+
+(test test-snapshot/utf-8
+  (snapshot *test-system*))
+
+(test test-start-2/utf-8
+  "Create a new prevalence system again"
+  (setf *test-system* (make-prevalence-system-w/format :utf-8))
+  (is-true *test-system*))
+
+(test test-restored/utf-8
+  (is (equal (get-string) *test-string/utf-8*)))
+
+(test test-start/latin-1
+  "Create a new prevalence system again with other charset"
+  (setf *test-system* (make-prevalence-system-w/format :latin-1))
+  (is-true *test-system*))
+
+(test test-restored/latin-1
+  (is-false (equal (get-string) *test-string/utf-8*)))
+  
