public const string __doc__ = "Fast object serialization/deserialization.\n\n"

+ "Differences from CPython:\n"

+ " - does not implement the undocumented fast mode\n";

[System.Runtime.CompilerServices.SpecialName]

public static void PerformModuleReload(PythonContext/*!*/ context, PythonDictionary/*!*/ dict) {

context.EnsureModuleException("PickleError", dict, "PickleError", "_pickle");

context.EnsureModuleException("PicklingError", dict, "PicklingError", "_pickle");

context.EnsureModuleException("UnpicklingError", dict, "UnpicklingError", "_pickle");

context.EnsureModuleException("UnpickleableError", dict, "UnpickleableError", "_pickle");

context.EnsureModuleException("BadPickleGet", dict, "BadPickleGet", "_pickle");

dict["__builtins__"] = context.BuiltinModuleInstance;

dict["compatible_formats"] = PythonOps.MakeList("1.0", "1.1", "1.2", "1.3", "2.0");

}

private static readonly PythonStruct.Struct _float64 = PythonStruct.Struct.Create(">d");

private const int highestProtocol = 2;

public const string __version__ = "1.71";

public const string format_version = "2.0";

public static int HIGHEST_PROTOCOL {

get { return highestProtocol; }

}

private const string Newline = "\n";

#region Public module-level functions

[Documentation("dump(obj, file, protocol=0) -> None\n\n"

+ "Pickle obj and write the result to file.\n"

+ "\n"

+ "See documentation for Pickler() for a description the file, protocol, and\n"

+ "(deprecated) bin parameters."

)]

public static void dump(CodeContext/*!*/ context, object obj, object file, object protocol=null, object bin=null) {

PicklerObject/*!*/ pickler = new PicklerObject(context, file, protocol, bin);

pickler.dump(context, obj);

}

[Documentation("dumps(obj, protocol=0) -> pickle string\n\n"

+ "Pickle obj and return the result as a string.\n"

+ "\n"

+ "See the documentation for Pickler() for a description of the protocol and\n"

+ "(deprecated) bin parameters."

)]

public static string dumps(CodeContext/*!*/ context, object obj, object protocol=null, object bin=null) {

//??? possible perf enhancement: use a C# TextWriter-backed IFileOutput and

// thus avoid Python call overhead. Also do similar thing for LoadFromString.

var stringIO = new StringBuilderOutput();

PicklerObject/*!*/ pickler = new PicklerObject(context, stringIO, protocol, bin);

pickler.dump(context, obj);

return stringIO.GetString();

}

[Documentation("load(file) -> unpickled object\n\n"

+ "Read pickle data from the open file object and return the corresponding\n"

+ "unpickled object. Data after the first pickle found is ignored, but the file\n"

+ "cursor is not reset, so if a file objects contains multiple pickles, then\n"

+ "load() may be called multiple times to unpickle them.\n"

+ "\n"

+ "file: an object (such as an open file or a StringIO) with read(num_chars) and\n"

+ " readline() methods that return strings\n"

+ "\n"

+ "load() automatically determines if the pickle data was written in binary or\n"

+ "text mode."

)]

public static object load(CodeContext/*!*/ context, object file) {

return new UnpicklerObject(context, file).load(context);

}

[Documentation("loads(string) -> unpickled object\n\n"

+ "Read a pickle object from a string, unpickle it, and return the resulting\n"

+ "reconstructed object. Characters in the string beyond the end of the first\n"

+ "pickle are ignored."

)]

public static object loads(CodeContext/*!*/ context, [BytesConversion]IList<byte> @string) {

return new UnpicklerObject(context, new PythonStringInput(PythonOps.MakeString(@string))).load(context);

}

#endregion

#region File I/O wrappers

/// <summary>

/// Interface for "file-like objects" that implement the protocol needed by load() and friends.

/// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.

/// </summary>

internal abstract class FileInput {

public abstract string Read(CodeContext/*!*/ context, int size);

public abstract string ReadLine(CodeContext/*!*/ context);

public virtual string ReadLineNoNewLine(CodeContext/*!*/ context) {

var raw = ReadLine(context);

return raw.Substring(0, raw.Length - 1);

}

public virtual char ReadChar(CodeContext context) {

string res = Read(context, 1);

if (res.Length < 1) {

throw PythonOps.EofError("unexpected EOF while unpickling");

}

return res[0];

}

public virtual int ReadInt(CodeContext context) {

return (int)ReadChar(context) |

((int)ReadChar(context)) << 8 |

((int)ReadChar(context)) << 16 |

((int)ReadChar(context)) << 24;

}

}

/// <summary>

/// Interface for "file-like objects" that implement the protocol needed by dump() and friends.

/// This enables the creation of thin wrappers that make fast .NET types and slow Python types look the same.

/// </summary>

internal abstract class FileOutput {

private readonly char[] int32chars = new char[4];

public abstract void Write(CodeContext/*!*/ context, string data);

public virtual void Write(CodeContext context, int data) {

int32chars[0] = (char)(int)((data & 0xff));

int32chars[1] = (char)(int)((data >> 8) & 0xff);

int32chars[2] = (char)(int)((data >> 16) & 0xff);

int32chars[3] = (char)(int)((data >> 24) & 0xff);

Write(context, new string(int32chars));

}

public virtual void Write(CodeContext context, char data) {

Write(context, ScriptingRuntimeHelpers.CharToString(data));

}

}

private class PythonFileInput : FileInput {

private object _readMethod;

private object _readLineMethod;

public PythonFileInput(CodeContext/*!*/ context, object file) {

if (!PythonOps.TryGetBoundAttr(context, file, "read", out _readMethod) ||

!PythonOps.IsCallable(context, _readMethod) ||

!PythonOps.TryGetBoundAttr(context, file, "readline", out _readLineMethod) ||

!PythonOps.IsCallable(context, _readLineMethod)

) {

throw PythonOps.TypeError("argument must have callable 'read' and 'readline' attributes");

}

}

public override string Read(CodeContext/*!*/ context, int size) {

return Converter.ConvertToString(PythonCalls.Call(context, _readMethod, size));

}

public override string ReadLine(CodeContext/*!*/ context) {

return Converter.ConvertToString(PythonCalls.Call(context, _readLineMethod));

}

}

internal class PythonStringInput : FileInput {

private readonly string _data;

int _offset;

public PythonStringInput(string data) {

_data = data;

}

public override string Read(CodeContext context, int size) {

var res = _data.Substring(_offset, size);

_offset += size;

return res;

}

public override string ReadLine(CodeContext context) {

return ReadLineWorker(true);

}

public override string ReadLineNoNewLine(CodeContext context) {

return ReadLineWorker(false);

}

public override char ReadChar(CodeContext context) {

if (_offset < _data.Length) {

return _data[_offset++];

}

throw PythonOps.EofError("unexpected EOF while unpickling");

}

public override int ReadInt(CodeContext context) {

if (_offset + 4 <= _data.Length) {

int res = _data[_offset]|

((int)_data[_offset + 1]) << 8 |

((int)_data[_offset + 2]) << 16 |

((int)_data[_offset + 3]) << 24;

_offset += 4;

return res;

}

throw PythonOps.EofError("unexpected EOF while unpickling");

}

private string ReadLineWorker(bool includeNewLine) {

string res;

for (int i = _offset; i < _data.Length; i++) {

if (_data[i] == '\n') {

res = _data.Substring(_offset, i - _offset + (includeNewLine ? 1 : 0));

_offset = i + 1;

return res;

}

}

res = _data.Substring(_offset);

_offset = _data.Length;

return res;

}

}

private class PythonFileLikeOutput : FileOutput {

private object _writeMethod;

public PythonFileLikeOutput(CodeContext/*!*/ context, object file) {

if (!PythonOps.TryGetBoundAttr(context, file, "write", out _writeMethod) ||

!PythonOps.IsCallable(context, this._writeMethod)

) {

throw PythonOps.TypeError("argument must have callable 'write' attribute");

}

}

public override void Write(CodeContext/*!*/ context, string data) {

PythonCalls.Call(context, _writeMethod, data);

}

}

private class PythonFileOutput : FileOutput {

private readonly PythonFile _file;

public PythonFileOutput(PythonFile file) {

_file = file;

}

public override void Write(CodeContext/*!*/ context, string data) {

_file.write(data);

}

}

private class StringBuilderOutput : FileOutput {

private readonly StringBuilder _builder = new StringBuilder(4096);

public string GetString() {

return _builder.ToString();

}

public override void Write(CodeContext context, char data) {

_builder.Append(data);

}

public override void Write(CodeContext context, int data) {

_builder.Append((char)(int)((data) & 0xff));

_builder.Append((char)(int)((data >> 8) & 0xff));

_builder.Append((char)(int)((data >> 16) & 0xff));

_builder.Append((char)(int)((data >> 24) & 0xff));

}

public override void Write(CodeContext context, string data) {

_builder.Append(data);

}

}

private class PythonReadableFileOutput : PythonFileLikeOutput {

private object _getValueMethod;

public PythonReadableFileOutput(CodeContext/*!*/ context, object file)

: base(context, file) {

if (!PythonOps.TryGetBoundAttr(context, file, "getvalue", out _getValueMethod) ||

!PythonOps.IsCallable(context, _getValueMethod)

) {

throw PythonOps.TypeError("argument must have callable 'getvalue' attribute");

}

}

public object GetValue(CodeContext/*!*/ context) {

return PythonCalls.Call(context, _getValueMethod);

}

}

#endregion

#region Opcode constants

internal static class Opcode {

public const char Append = 'a';

public const char Appends = 'e';

public const char BinFloat = 'G';

public const char BinGet = 'h';

public const char BinInt = 'J';

public const char BinInt1 = 'K';

public const char BinInt2 = 'M';

public const char BinPersid = 'Q';

public const char BinPut = 'q';

public const char BinString = 'T';

public const char BinUnicode = 'X';

public const char Build = 'b';

public const char Dict = 'd';

public const char Dup = '2';

public const char EmptyDict = '}';

public const char EmptyList = ']';

public const char EmptyTuple = ')';

public const char Ext1 = '\x82';

public const char Ext2 = '\x83';

public const char Ext4 = '\x84';

public const char Float = 'F';

public const char Get = 'g';

public const char Global = 'c';

public const char Inst = 'i';

public const char Int = 'I';

public const char List = 'l';

public const char Long = 'L';

public const char Long1 = '\x8a';

public const char Long4 = '\x8b';

public const char LongBinGet = 'j';

public const char LongBinPut = 'r';

public const char Mark = '(';

public const char NewFalse = '\x89';

public const char NewObj = '\x81';

public const char NewTrue = '\x88';

public const char NoneValue = 'N';

public const char Obj = 'o';

public const char PersId = 'P';

public const char Pop = '0';

public const char PopMark = '1';

public const char Proto = '\x80';

public const char Put = 'p';

public const char Reduce = 'R';

public const char SetItem = 's';

public const char SetItems = 'u';

public const char ShortBinstring = 'U';

public const char Stop = '.';

public const char String = 'S';

public const char Tuple = 't';

public const char Tuple1 = '\x85';

public const char Tuple2 = '\x86';

public const char Tuple3 = '\x87';

public const char Unicode = 'V';

}

#endregion

#region Pickler object

public static PicklerObject/*!*/ Pickler(CodeContext/*!*/ context, object file=null, object protocol=null, object bin=null) {

return new PicklerObject(context, file, protocol, bin);

}

[Documentation("Pickler(file, protocol=0) -> Pickler object\n\n"

+ "A Pickler object serializes Python objects to a pickle bytecode stream, which\n"

+ "can then be converted back into equivalent objects using an Unpickler.\n"

+ "\n"

+ "file: an object (such as an open file) that has a write(string) method.\n"

+ "protocol: if omitted, protocol 0 is used. If HIGHEST_PROTOCOL or a negative\n"

+ " number, the highest available protocol is used.\n"

+ "bin: (deprecated; use protocol instead) for backwards compability, a 'bin'\n"

+ " keyword parameter is supported. When protocol is specified it is ignored.\n"

+ " If protocol is not specified, then protocol 0 is used if bin is false, and\n"

+ " protocol 1 is used if bin is true."

)]

[PythonType("Pickler"), PythonHidden]

public class PicklerObject {

private const char LowestPrintableChar = (char)32;

private const char HighestPrintableChar = (char)126;

// max elements that can be set/appended at a time using SETITEMS/APPENDS

private delegate void PickleFunction(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object value);

private static readonly Dictionary<Type, PickleFunction> _dispatchTable;

private int _batchSize = 1000;

private FileOutput _file;

private int _protocol;

private PythonDictionary _memo; // memo if the user accesses the memo property

private Dictionary<object, int> _privMemo; // internal fast memo which we can use if the user doesn't access memo

private object _persist_id;

static PicklerObject() {

_dispatchTable = new Dictionary<Type, PickleFunction>();

_dispatchTable[typeof(PythonDictionary)] = SaveDict;

_dispatchTable[typeof(PythonTuple)] = SaveTuple;

_dispatchTable[typeof(PythonList)] = SaveList;

_dispatchTable[typeof(PythonFunction)] = SaveGlobal;

_dispatchTable[typeof(BuiltinFunction)] = SaveGlobal;

_dispatchTable[typeof(PythonType)] = SaveGlobal;

}

#region Public API

[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]

public PythonDictionary memo {

get {

if (_memo == null) {

// create publicly viewable memo

PythonDictionary resMemo = new PythonDictionary();

foreach (var v in _privMemo) {

resMemo._storage.AddNoLock(

ref resMemo._storage,

Builtin.id(v.Key),

PythonTuple.MakeTuple(v.Value, v.Key)

);

}

_memo = resMemo;

}

return _memo;

}

set {

_memo = value;

_privMemo = null;

}

}

public int proto {

get { return _protocol; }

set { _protocol = value; }

}

public int _BATCHSIZE {

get { return _batchSize; }

set { _batchSize = value; }

}

public object persistent_id {

get {

return _persist_id;

}

set {

_persist_id = value;

}

}

public int binary {

get { return _protocol == 0 ? 1 : 0; }

set { _protocol = value; }

}

public int fast {

// We don't implement fast, but we silently ignore it when it's set so that test_cpickle works.

// For a description of fast, see http://mail.python.org/pipermail/python-bugs-list/2001-October/007695.html

get { return 0; }

set { /* ignore */ }

}

public PicklerObject(CodeContext/*!*/ context, object file, object protocol, object bin) {

int intProtocol;

if (file == null) {

_file = new PythonReadableFileOutput(context, new PythonIOModule.StringIO(context, "", "\n"));

} else if (Converter.TryConvertToInt32(file, out intProtocol)) {

// For undocumented (yet tested in official CPython tests) list-based pickler, the

// user could do something like Pickler(1), which would create a protocol-1 pickler

// with an internal string output buffer (retrievable using getvalue()). For a little

// more info, see

// https://sourceforge.net/tracker/?func=detail&atid=105470&aid=939395&group_id=5470

_file = new PythonReadableFileOutput(context, new PythonIOModule.StringIO(context, "", "\n"));

protocol = file;

} else if (file is PythonFile) {

_file = new PythonFileOutput((PythonFile)file);

} else if (file is FileOutput) {

_file = (FileOutput)file;

} else {

_file = new PythonFileLikeOutput(context, file);

}

_privMemo = new Dictionary<object, int>(256, ReferenceEqualityComparer.Instance);

if (protocol == null) protocol = PythonOps.IsTrue(bin) ? 1 : 0;

intProtocol = context.LanguageContext.ConvertToInt32(protocol);

if (intProtocol > highestProtocol) {

throw PythonOps.ValueError("pickle protocol {0} asked for; the highest available protocol is {1}", intProtocol, highestProtocol);

} else if (intProtocol < 0) {

this._protocol = highestProtocol;

} else {

this._protocol = intProtocol;

}

}

[Documentation("dump(obj) -> None\n\n"

+ "Pickle obj and write the result to the file object that was passed to the\n"

+ "constructor\n."

+ "\n"

+ "Note that you may call dump() multiple times to pickle multiple objects. To\n"

+ "unpickle the stream, you will need to call Unpickler's load() method a\n"

+ "corresponding number of times.\n"

+ "\n"

+ "The first time a particular object is encountered, it will be pickled normally.\n"

+ "If the object is encountered again (in the same or a later dump() call), a\n"

+ "reference to the previously generated value will be pickled. Unpickling will\n"

+ "then create multiple references to a single object."

)]

public void dump(CodeContext/*!*/ context, object obj) {

if (_protocol >= 2) WriteProto(context);

Save(context, obj);

Write(context, Opcode.Stop);

}

[Documentation("clear_memo() -> None\n\n"

+ "Clear the memo, which is used internally by the pickler to keep track of which\n"

+ "objects have already been pickled (so that shared or recursive objects are\n"

+ "pickled only once)."

)]

public void clear_memo() {

if (_memo != null) {

_memo.Clear();

} else {

_privMemo.Clear();

}

}

private void Memoize(object obj) {

if (_memo != null) {

if (!MemoContains(PythonOps.Id(obj))) {

_memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(_memo.Count, obj);

}

} else {

if(!_privMemo.ContainsKey(obj)) {

_privMemo[obj] = _privMemo.Count;

}

}

}

private int MemoizeNew(object obj) {

int res;

if (_memo != null) {

Debug.Assert(!_memo.ContainsKey(obj));

_memo[PythonOps.Id(obj)] = PythonTuple.MakeTuple(res = _memo.Count, obj);

} else {

Debug.Assert(!_privMemo.ContainsKey(obj));

_privMemo[obj] = res = _privMemo.Count;

}

return res;

}

private bool MemoContains(object obj) {

if (_memo != null) {

return _memo.Contains(PythonOps.Id(obj));

}

return _privMemo.ContainsKey(obj);

}

private bool TryWriteFastGet(CodeContext context, object obj) {

int value;

if (_memo != null) {

return TryWriteSlowGet(context, obj);

} else if (_privMemo.TryGetValue(obj, out value)) {

WriteGetOrPut(context, true, value);

return true;

}

return false;

}

private bool TryWriteSlowGet(CodeContext context, object obj) {

object value;

if (_memo.TryGetValue(obj, out value)) {

WriteGetOrPut(context, true, (PythonTuple)value);

return true;

}

return false;

}

[Documentation("getvalue() -> string\n\n"

+ "Return the value of the internal string. Raises PicklingError if a file object\n"

+ "was passed to this pickler's constructor."

)]

public object getvalue(CodeContext/*!*/ context) {

if (_file is PythonReadableFileOutput) {

return ((PythonReadableFileOutput)_file).GetValue(context);

}

throw PythonExceptions.CreateThrowable(PicklingError(context), "Attempt to getvalue() a non-list-based pickler");

}

#endregion

#region Save functions

private void Save(CodeContext/*!*/ context, object obj) {

if (_persist_id == null || !TrySavePersistId(context, obj)) {

PickleFunction pickleFunction;

// several typees are never memoized, check for these first.

if (obj == null) {

SaveNone(this, context, obj);

} else if (obj is int) {

SaveInteger(this, context, obj);

} else if(obj is BigInteger) {

SaveLong(this, context, obj);

} else if (obj is bool) {

SaveBoolean(this, context, obj);

} else if (obj is double) {

SaveFloat(this, context, obj);

} else if(!TryWriteFastGet(context, obj)) {

if (obj is string) {

// strings are common, specialize them.

SaveUnicode(this, context, obj);

} else {

if (!_dispatchTable.TryGetValue(obj.GetType(), out pickleFunction)) {

if (obj is PythonType) {

// treat classes with metaclasses like regular classes

pickleFunction = SaveGlobal;

} else {

pickleFunction = SaveObject;

}

}

pickleFunction(this, context, obj);

}

}

}

}

private bool TrySavePersistId(CodeContext context, object obj) {

Debug.Assert(_persist_id != null);

string res = Converter.ConvertToString(context.LanguageContext.CallSplat(_persist_id, obj));

if (res != null) {

SavePersId(context, res);

return true;

}

return false;

}

private void SavePersId(CodeContext/*!*/ context, string res) {

if (this.binary != 0) {

Save(context, res);

Write(context, Opcode.BinPersid);

} else {

Write(context, Opcode.PersId);

Write(context, res);

Write(context, "\n");

}

}

private static void SaveBoolean(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {

Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Boolean), "arg must be bool");

if (pickler._protocol < 2) {

pickler.Write(context, Opcode.Int);

pickler.Write(context, String.Format("0{0}", ((bool)obj) ? 1 : 0));

pickler.Write(context, Newline);

} else {

if ((bool)obj) {

pickler.Write(context, Opcode.NewTrue);

} else {

pickler.Write(context, Opcode.NewFalse);

}

}

}

private static void SaveDict(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {

Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Dict), "arg must be dict");

Debug.Assert(!pickler.MemoContains(obj));

int index = pickler.MemoizeNew(obj);

if (pickler._protocol < 1) {

pickler.Write(context, Opcode.Mark);

pickler.Write(context, Opcode.Dict);

} else {

pickler.Write(context, Opcode.EmptyDict);

}

pickler.WritePut(context, index);

pickler.BatchSetItems(context, (PythonDictionary)obj);

}

private static void SaveFloat(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {

Debug.Assert(DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Double), "arg must be float");

if (pickler._protocol < 1) {

pickler.Write(context, Opcode.Float);

pickler.WriteFloatAsString(context, obj);

} else {

pickler.Write(context, Opcode.BinFloat);

pickler.WriteFloat64(context, obj);

}

}

private static void SaveGlobal(PicklerObject/*!*/ pickler, CodeContext/*!*/ context, object obj) {

Debug.Assert(

DynamicHelpers.GetPythonType(obj).Equals(TypeCache.Function) ||

DynamicHelpers.GetPythonType(obj).Equals(TypeCache.BuiltinFunction) ||

DynamicHelpers.GetPythonType(obj).Equals(TypeCache.PythonType) ||

DynamicHelpers.GetPythonType(obj).IsSubclassOf(TypeCache.PythonType),

"arg must be classic class, function, built-in function or method, or new-style type"

);

PythonType pt = obj as PythonType;

if (pt != null) {

pickler.SaveGlobalByName(context, obj, pt.Name);

} else {

object name;

if (PythonOps.TryGetBoundAttr(context, obj, "__name__", out name)) {

pickler.SaveGlobalByName(context, obj, name);

} else {

throw pickler.CannotPickle(context, obj, "could not determine its __name__");

}

}

}

private void SaveGlobalByName(CodeContext/*!*/ context, object obj, object name) {

Debug.Assert(!MemoContains(obj));

object moduleName = FindModuleForGlobal(context, obj, name);

if (_protocol >= 2) {

object code;

if (PythonCopyReg.GetExtensionRegistry(context).TryGetValue(PythonTuple.MakeTuple(moduleName, name), out code)) {

if (IsUInt8(context, code)) {

Write(context, Opcode.Ext1);

WriteUInt8(context, code);

} else if (IsUInt16(context, code)) {

Write(context, Opcode.Ext2);

WriteUInt16(context, code);

} else if (IsInt32(context, code)) {

Write(context, Opcode.Ext4);

WriteInt32(context, code);