using System; using System.Buffers; using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Resources; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Security; using System.Security.Cryptography; using System.Security.Permissions; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using LiteDB.Engine; using LiteDB.Utils; using LiteDB.Utils.Extensions; using Microsoft.CodeAnalysis; [assembly: CompilationRelaxations(8)] [assembly: RuntimeCompatibility(WrapNonExceptionThrows = true)] [assembly: Debuggable(DebuggableAttribute.DebuggingModes.IgnoreSymbolStoreSequencePoints)] [assembly: InternalsVisibleTo("LiteDB.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010029e66990e22110ce40a7197e37f8f82df3332c399e696df7f27d09e14ee590ac2dda735d4777fe554c427540bde93b14d3d26c04731c963383dcaa18859c8cbcd4a1a9c394d1204f474c2ab6f23a2eaadf81eb8a7a3d3cc73658868b0302163b92a2614ca050ab703be33c3e1d76f55b11f4f87cb73558f3aa69c1ce726d9ee8")] [assembly: TargetFramework(".NETStandard,Version=v2.0", FrameworkDisplayName = ".NET Standard 2.0")] [assembly: AssemblyCompany("MaurĂ­cio David")] [assembly: AssemblyConfiguration("Release")] [assembly: AssemblyCopyright("MIT")] [assembly: AssemblyDescription("LiteDB - A lightweight embedded .NET NoSQL document store in a single datafile")] [assembly: AssemblyFileVersion("5.0.21")] [assembly: AssemblyInformationalVersion("5.0.21+391cc9318c5be6e56cb71b3ce14b1ed9cb324763")] [assembly: AssemblyProduct("LiteDB")] [assembly: AssemblyTitle("LiteDB")] [assembly: AssemblyMetadata("RepositoryUrl", "https://github.com/mbdavid/LiteDB")] [assembly: NeutralResourcesLanguage("en-US")] [assembly: SecurityPermission(SecurityAction.RequestMinimum, SkipVerification = true)] [assembly: AssemblyVersion("5.0.21.0")] [module: UnverifiableCode] [module: RefSafetyRules(11)] namespace Microsoft.CodeAnalysis { [CompilerGenerated] [Microsoft.CodeAnalysis.Embedded] internal sealed class EmbeddedAttribute : Attribute { } } namespace System.Runtime.CompilerServices { [CompilerGenerated] [Microsoft.CodeAnalysis.Embedded] internal sealed class IsReadOnlyAttribute : Attribute { } [CompilerGenerated] [Microsoft.CodeAnalysis.Embedded] [AttributeUsage(AttributeTargets.Class | AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Event | AttributeTargets.Parameter | AttributeTargets.ReturnValue | AttributeTargets.GenericParameter, AllowMultiple = false, Inherited = false)] internal sealed class NullableAttribute : Attribute { public readonly byte[] NullableFlags; public NullableAttribute(byte P_0) { NullableFlags = new byte[1] { P_0 }; } public NullableAttribute(byte[] P_0) { NullableFlags = P_0; } } [CompilerGenerated] [Microsoft.CodeAnalysis.Embedded] [AttributeUsage(AttributeTargets.Class | AttributeTargets.Struct | AttributeTargets.Method | AttributeTargets.Interface | AttributeTargets.Delegate, AllowMultiple = false, Inherited = false)] internal sealed class NullableContextAttribute : Attribute { public readonly byte Flag; public NullableContextAttribute(byte P_0) { Flag = P_0; } } [CompilerGenerated] [Microsoft.CodeAnalysis.Embedded] [AttributeUsage(AttributeTargets.Module, AllowMultiple = false, Inherited = false)] internal sealed class RefSafetyRulesAttribute : Attribute { public readonly int Version; public RefSafetyRulesAttribute(int P_0) { Version = P_0; } } } namespace LiteDB { public sealed class LiteCollection : ILiteCollection { private readonly string _collection; private readonly ILiteEngine _engine; private readonly List _includes; private readonly BsonMapper _mapper; private readonly EntityMapper _entity; private readonly MemberMapper _id; private readonly BsonAutoId _autoId; public string Name => _collection; public BsonAutoId AutoId => _autoId; public EntityMapper EntityMapper => _entity; public int Count() { return Query().Count(); } public int Count(BsonExpression predicate) { if (predicate == null) { throw new ArgumentNullException("predicate"); } return Query().Where(predicate).Count(); } public int Count(string predicate, BsonDocument parameters) { return Count(BsonExpression.Create(predicate, parameters)); } public int Count(string predicate, params BsonValue[] args) { return Count(BsonExpression.Create(predicate, args)); } public int Count(Expression> predicate) { return Count(_mapper.GetExpression(predicate)); } public int Count(Query query) { return new LiteQueryable(_engine, _mapper, _collection, query).Count(); } public long LongCount() { return Query().LongCount(); } public long LongCount(BsonExpression predicate) { if (predicate == null) { throw new ArgumentNullException("predicate"); } return Query().Where(predicate).LongCount(); } public long LongCount(string predicate, BsonDocument parameters) { return LongCount(BsonExpression.Create(predicate, parameters)); } public long LongCount(string predicate, params BsonValue[] args) { return LongCount(BsonExpression.Create(predicate, args)); } public long LongCount(Expression> predicate) { return LongCount(_mapper.GetExpression(predicate)); } public long LongCount(Query query) { return new LiteQueryable(_engine, _mapper, _collection, query).Count(); } public bool Exists(BsonExpression predicate) { if (predicate == null) { throw new ArgumentNullException("predicate"); } return Query().Where(predicate).Exists(); } public bool Exists(string predicate, BsonDocument parameters) { return Exists(BsonExpression.Create(predicate, parameters)); } public bool Exists(string predicate, params BsonValue[] args) { return Exists(BsonExpression.Create(predicate, args)); } public bool Exists(Expression> predicate) { return Exists(_mapper.GetExpression(predicate)); } public bool Exists(Query query) { return new LiteQueryable(_engine, _mapper, _collection, query).Exists(); } public BsonValue Min(BsonExpression keySelector) { if (string.IsNullOrEmpty(keySelector)) { throw new ArgumentNullException("keySelector"); } BsonDocument bsonDocument = Query().OrderBy(keySelector).Select(keySelector).ToDocuments() .First(); return bsonDocument[bsonDocument.Keys.First()]; } public BsonValue Min() { return Min("_id"); } public K Min(Expression> keySelector) { if (keySelector == null) { throw new ArgumentNullException("keySelector"); } BsonExpression expression = _mapper.GetExpression(keySelector); BsonValue value = Min(expression); return (K)_mapper.Deserialize(typeof(K), value); } public BsonValue Max(BsonExpression keySelector) { if (string.IsNullOrEmpty(keySelector)) { throw new ArgumentNullException("keySelector"); } BsonDocument bsonDocument = Query().OrderByDescending(keySelector).Select(keySelector).ToDocuments() .First(); return bsonDocument[bsonDocument.Keys.First()]; } public BsonValue Max() { return Max("_id"); } public K Max(Expression> keySelector) { if (keySelector == null) { throw new ArgumentNullException("keySelector"); } BsonExpression expression = _mapper.GetExpression(keySelector); BsonValue value = Max(expression); return (K)_mapper.Deserialize(typeof(K), value); } public bool Delete(BsonValue id) { if (id == null || id.IsNull) { throw new ArgumentNullException("id"); } return _engine.Delete(_collection, new BsonValue[1] { id }) == 1; } public int DeleteAll() { return _engine.DeleteMany(_collection, null); } public int DeleteMany(BsonExpression predicate) { if (predicate == null) { throw new ArgumentNullException("predicate"); } return _engine.DeleteMany(_collection, predicate); } public int DeleteMany(string predicate, BsonDocument parameters) { return DeleteMany(BsonExpression.Create(predicate, parameters)); } public int DeleteMany(string predicate, params BsonValue[] args) { return DeleteMany(BsonExpression.Create(predicate, args)); } public int DeleteMany(Expression> predicate) { return DeleteMany(_mapper.GetExpression(predicate)); } public ILiteQueryable Query() { return new LiteQueryable(_engine, _mapper, _collection, new Query()).Include(_includes); } public IEnumerable Find(BsonExpression predicate, int skip = 0, int limit = int.MaxValue) { if (predicate == null) { throw new ArgumentNullException("predicate"); } return Query().Include(_includes).Where(predicate).Skip(skip) .Limit(limit) .ToEnumerable(); } public IEnumerable Find(Query query, int skip = 0, int limit = int.MaxValue) { if (query == null) { throw new ArgumentNullException("query"); } if (skip != 0) { query.Offset = skip; } if (limit != int.MaxValue) { query.Limit = limit; } return new LiteQueryable(_engine, _mapper, _collection, query).ToEnumerable(); } public IEnumerable Find(Expression> predicate, int skip = 0, int limit = int.MaxValue) { return Find(_mapper.GetExpression(predicate), skip, limit); } public T FindById(BsonValue id) { if (id == null || id.IsNull) { throw new ArgumentNullException("id"); } return Find(BsonExpression.Create("_id = @0", id)).FirstOrDefault(); } public T FindOne(BsonExpression predicate) { return Find(predicate).FirstOrDefault(); } public T FindOne(string predicate, BsonDocument parameters) { return FindOne(BsonExpression.Create(predicate, parameters)); } public T FindOne(BsonExpression predicate, params BsonValue[] args) { return FindOne(BsonExpression.Create(predicate, args)); } public T FindOne(Expression> predicate) { return FindOne(_mapper.GetExpression(predicate)); } public T FindOne(Query query) { return Find(query).FirstOrDefault(); } public IEnumerable FindAll() { return Query().Include(_includes).ToEnumerable(); } public ILiteCollection Include(Expression> keySelector) { if (keySelector == null) { throw new ArgumentNullException("keySelector"); } BsonExpression expression = _mapper.GetExpression(keySelector); return Include(expression); } public ILiteCollection Include(BsonExpression keySelector) { if (string.IsNullOrEmpty(keySelector)) { throw new ArgumentNullException("keySelector"); } LiteCollection liteCollection = new LiteCollection(_collection, _autoId, _engine, _mapper); liteCollection._includes.AddRange(_includes); liteCollection._includes.Add(keySelector); return liteCollection; } public bool EnsureIndex(string name, BsonExpression expression, bool unique = false) { if (string.IsNullOrEmpty(name)) { throw new ArgumentNullException("name"); } if (expression == null) { throw new ArgumentNullException("expression"); } return _engine.EnsureIndex(_collection, name, expression, unique); } public bool EnsureIndex(BsonExpression expression, bool unique = false) { if (expression == null) { throw new ArgumentNullException("expression"); } string name = Regex.Replace(expression.Source, "[^a-z0-9]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled); return EnsureIndex(name, expression, unique); } public bool EnsureIndex(Expression> keySelector, bool unique = false) { BsonExpression indexExpression = GetIndexExpression(keySelector); return EnsureIndex(indexExpression, unique); } public bool EnsureIndex(string name, Expression> keySelector, bool unique = false) { BsonExpression indexExpression = GetIndexExpression(keySelector); return EnsureIndex(name, indexExpression, unique); } private BsonExpression GetIndexExpression(Expression> keySelector) { BsonExpression bsonExpression = _mapper.GetIndexExpression(keySelector); if (typeof(K).IsEnumerable() && bsonExpression.IsScalar) { if (bsonExpression.Type != BsonExpressionType.Path) { throw new LiteException(0, "Expression `" + bsonExpression.Source + "` must return a enumerable expression"); } bsonExpression = bsonExpression.Source + "[*]"; } return bsonExpression; } public bool DropIndex(string name) { return _engine.DropIndex(_collection, name); } public BsonValue Insert(T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } BsonDocument bsonDocument = _mapper.ToDocument(entity); bool flag = RemoveDocId(bsonDocument); _engine.Insert(_collection, new BsonDocument[1] { bsonDocument }, _autoId); BsonValue bsonValue = bsonDocument["_id"]; if (flag) { _id.Setter(entity, bsonValue.RawValue); } return bsonValue; } public void Insert(BsonValue id, T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } if (id == null || id.IsNull) { throw new ArgumentNullException("id"); } BsonDocument bsonDocument = _mapper.ToDocument(entity); bsonDocument["_id"] = id; _engine.Insert(_collection, new BsonDocument[1] { bsonDocument }, _autoId); } public int Insert(IEnumerable entities) { if (entities == null) { throw new ArgumentNullException("entities"); } return _engine.Insert(_collection, GetBsonDocs(entities), _autoId); } [Obsolete("Use normal Insert()")] public int InsertBulk(IEnumerable entities, int batchSize = 5000) { if (entities == null) { throw new ArgumentNullException("entities"); } return _engine.Insert(_collection, GetBsonDocs(entities), _autoId); } private IEnumerable GetBsonDocs(IEnumerable documents) { foreach (T document in documents) { BsonDocument doc = _mapper.ToDocument(document); bool removed = RemoveDocId(doc); yield return doc; if (removed && _id != null) { _id.Setter(document, doc["_id"].RawValue); } } } private bool RemoveDocId(BsonDocument doc) { if (_id != null && doc.TryGetValue("_id", out var value) && ((_autoId == BsonAutoId.Int32 && value.IsInt32 && value.AsInt32 == 0) || (_autoId == BsonAutoId.ObjectId && (value.IsNull || (value.IsObjectId && value.AsObjectId == ObjectId.Empty))) || (_autoId == BsonAutoId.Guid && value.IsGuid && value.AsGuid == Guid.Empty) || (_autoId == BsonAutoId.Int64 && value.IsInt64 && value.AsInt64 == 0L))) { doc.Remove("_id"); return true; } return false; } public bool Update(T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } BsonDocument bsonDocument = _mapper.ToDocument(entity); return _engine.Update(_collection, new BsonDocument[1] { bsonDocument }) > 0; } public bool Update(BsonValue id, T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } if (id == null || id.IsNull) { throw new ArgumentNullException("id"); } BsonDocument bsonDocument = _mapper.ToDocument(entity); bsonDocument["_id"] = id; return _engine.Update(_collection, new BsonDocument[1] { bsonDocument }) > 0; } public int Update(IEnumerable entities) { if (entities == null) { throw new ArgumentNullException("entities"); } return _engine.Update(_collection, entities.Select((T x) => _mapper.ToDocument(x))); } public int UpdateMany(BsonExpression transform, BsonExpression predicate) { if (transform == null) { throw new ArgumentNullException("transform"); } if (predicate == null) { throw new ArgumentNullException("predicate"); } if (transform.Type != BsonExpressionType.Document) { throw new ArgumentException("Extend expression must return a document. Eg: `col.UpdateMany('{ Name: UPPER(Name) }', 'Age > 10')`"); } return _engine.UpdateMany(_collection, transform, predicate); } public int UpdateMany(Expression> extend, Expression> predicate) { if (extend == null) { throw new ArgumentNullException("extend"); } if (predicate == null) { throw new ArgumentNullException("predicate"); } BsonExpression expression = _mapper.GetExpression(extend); BsonExpression expression2 = _mapper.GetExpression(predicate); if (expression.Type != BsonExpressionType.Document) { throw new ArgumentException("Extend expression must return an anonymous class to be merge with entities. Eg: `col.UpdateMany(x => new { Name = x.Name.ToUpper() }, x => x.Age > 10)`"); } return _engine.UpdateMany(_collection, expression, expression2); } public bool Upsert(T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } return Upsert(new T[1] { entity }) == 1; } public int Upsert(IEnumerable entities) { if (entities == null) { throw new ArgumentNullException("entities"); } return _engine.Upsert(_collection, GetBsonDocs(entities), _autoId); } public bool Upsert(BsonValue id, T entity) { if (entity == null) { throw new ArgumentNullException("entity"); } if (id == null || id.IsNull) { throw new ArgumentNullException("id"); } BsonDocument bsonDocument = _mapper.ToDocument(entity); bsonDocument["_id"] = id; return _engine.Upsert(_collection, new BsonDocument[1] { bsonDocument }, _autoId) > 0; } internal LiteCollection(string name, BsonAutoId autoId, ILiteEngine engine, BsonMapper mapper) { _collection = name ?? mapper.ResolveCollectionName(typeof(T)); _engine = engine; _mapper = mapper; _includes = new List(); if (typeof(T) == typeof(BsonDocument)) { _entity = null; _id = null; _autoId = autoId; return; } _entity = mapper.GetEntityMapper(typeof(T)); _id = _entity.Id; if (_id != null && _id.AutoId) { _autoId = ((_id.DataType == typeof(int) || _id.DataType == typeof(int?)) ? BsonAutoId.Int32 : ((_id.DataType == typeof(long) || _id.DataType == typeof(long?)) ? BsonAutoId.Int64 : ((_id.DataType == typeof(Guid) || _id.DataType == typeof(Guid?)) ? BsonAutoId.Guid : BsonAutoId.ObjectId))); } else { _autoId = autoId; } } } public interface ILiteCollection { string Name { get; } BsonAutoId AutoId { get; } EntityMapper EntityMapper { get; } ILiteCollection Include(Expression> keySelector); ILiteCollection Include(BsonExpression keySelector); bool Upsert(T entity); int Upsert(IEnumerable entities); bool Upsert(BsonValue id, T entity); bool Update(T entity); bool Update(BsonValue id, T entity); int Update(IEnumerable entities); int UpdateMany(BsonExpression transform, BsonExpression predicate); int UpdateMany(Expression> extend, Expression> predicate); BsonValue Insert(T entity); void Insert(BsonValue id, T entity); int Insert(IEnumerable entities); int InsertBulk(IEnumerable entities, int batchSize = 5000); bool EnsureIndex(string name, BsonExpression expression, bool unique = false); bool EnsureIndex(BsonExpression expression, bool unique = false); bool EnsureIndex(Expression> keySelector, bool unique = false); bool EnsureIndex(string name, Expression> keySelector, bool unique = false); bool DropIndex(string name); ILiteQueryable Query(); IEnumerable Find(BsonExpression predicate, int skip = 0, int limit = int.MaxValue); IEnumerable Find(Query query, int skip = 0, int limit = int.MaxValue); IEnumerable Find(Expression> predicate, int skip = 0, int limit = int.MaxValue); T FindById(BsonValue id); T FindOne(BsonExpression predicate); T FindOne(string predicate, BsonDocument parameters); T FindOne(BsonExpression predicate, params BsonValue[] args); T FindOne(Expression> predicate); T FindOne(Query query); IEnumerable FindAll(); bool Delete(BsonValue id); int DeleteAll(); int DeleteMany(BsonExpression predicate); int DeleteMany(string predicate, BsonDocument parameters); int DeleteMany(string predicate, params BsonValue[] args); int DeleteMany(Expression> predicate); int Count(); int Count(BsonExpression predicate); int Count(string predicate, BsonDocument parameters); int Count(string predicate, params BsonValue[] args); int Count(Expression> predicate); int Count(Query query); long LongCount(); long LongCount(BsonExpression predicate); long LongCount(string predicate, BsonDocument parameters); long LongCount(string predicate, params BsonValue[] args); long LongCount(Expression> predicate); long LongCount(Query query); bool Exists(BsonExpression predicate); bool Exists(string predicate, BsonDocument parameters); bool Exists(string predicate, params BsonValue[] args); bool Exists(Expression> predicate); bool Exists(Query query); BsonValue Min(BsonExpression keySelector); BsonValue Min(); K Min(Expression> keySelector); BsonValue Max(BsonExpression keySelector); BsonValue Max(); K Max(Expression> keySelector); } public interface ILiteDatabase : IDisposable { BsonMapper Mapper { get; } ILiteStorage FileStorage { get; } int UserVersion { get; set; } TimeSpan Timeout { get; set; } bool UtcDate { get; set; } long LimitSize { get; set; } int CheckpointSize { get; set; } Collation Collation { get; } ILiteCollection GetCollection(string name, BsonAutoId autoId = BsonAutoId.ObjectId); ILiteCollection GetCollection(); ILiteCollection GetCollection(BsonAutoId autoId); ILiteCollection GetCollection(string name, BsonAutoId autoId = BsonAutoId.ObjectId); bool BeginTrans(); bool Commit(); bool Rollback(); ILiteStorage GetStorage(string filesCollection = "_files", string chunksCollection = "_chunks"); IEnumerable GetCollectionNames(); bool CollectionExists(string name); bool DropCollection(string name); bool RenameCollection(string oldName, string newName); IBsonDataReader Execute(TextReader commandReader, BsonDocument parameters = null); IBsonDataReader Execute(string command, BsonDocument parameters = null); IBsonDataReader Execute(string command, params BsonValue[] args); void Checkpoint(); long Rebuild(RebuildOptions options = null); BsonValue Pragma(string name); BsonValue Pragma(string name, BsonValue value); } public interface ILiteQueryable : ILiteQueryableResult { ILiteQueryable Include(BsonExpression path); ILiteQueryable Include(List paths); ILiteQueryable Include(Expression> path); ILiteQueryable Where(BsonExpression predicate); ILiteQueryable Where(string predicate, BsonDocument parameters); ILiteQueryable Where(string predicate, params BsonValue[] args); ILiteQueryable Where(Expression> predicate); ILiteQueryable OrderBy(BsonExpression keySelector, int order = 1); ILiteQueryable OrderBy(Expression> keySelector, int order = 1); ILiteQueryable OrderByDescending(BsonExpression keySelector); ILiteQueryable OrderByDescending(Expression> keySelector); ILiteQueryable GroupBy(BsonExpression keySelector); ILiteQueryable Having(BsonExpression predicate); ILiteQueryableResult Select(BsonExpression selector); ILiteQueryableResult Select(Expression> selector); } public interface ILiteQueryableResult { ILiteQueryableResult Limit(int limit); ILiteQueryableResult Skip(int offset); ILiteQueryableResult Offset(int offset); ILiteQueryableResult ForUpdate(); BsonDocument GetPlan(); IBsonDataReader ExecuteReader(); IEnumerable ToDocuments(); IEnumerable ToEnumerable(); List ToList(); T[] ToArray(); int Into(string newCollection, BsonAutoId autoId = BsonAutoId.ObjectId); T First(); T FirstOrDefault(); T Single(); T SingleOrDefault(); int Count(); long LongCount(); bool Exists(); } public interface ILiteRepository : IDisposable { ILiteDatabase Database { get; } BsonValue Insert(T entity, string collectionName = null); int Insert(IEnumerable entities, string collectionName = null); bool Update(T entity, string collectionName = null); int Update(IEnumerable entities, string collectionName = null); bool Upsert(T entity, string collectionName = null); int Upsert(IEnumerable entities, string collectionName = null); bool Delete(BsonValue id, string collectionName = null); int DeleteMany(BsonExpression predicate, string collectionName = null); int DeleteMany(Expression> predicate, string collectionName = null); ILiteQueryable Query(string collectionName = null); bool EnsureIndex(string name, BsonExpression expression, bool unique = false, string collectionName = null); bool EnsureIndex(BsonExpression expression, bool unique = false, string collectionName = null); bool EnsureIndex(Expression> keySelector, bool unique = false, string collectionName = null); bool EnsureIndex(string name, Expression> keySelector, bool unique = false, string collectionName = null); T SingleById(BsonValue id, string collectionName = null); List Fetch(BsonExpression predicate, string collectionName = null); List Fetch(Expression> predicate, string collectionName = null); T First(BsonExpression predicate, string collectionName = null); T First(Expression> predicate, string collectionName = null); T FirstOrDefault(BsonExpression predicate, string collectionName = null); T FirstOrDefault(Expression> predicate, string collectionName = null); T Single(BsonExpression predicate, string collectionName = null); T Single(Expression> predicate, string collectionName = null); T SingleOrDefault(BsonExpression predicate, string collectionName = null); T SingleOrDefault(Expression> predicate, string collectionName = null); } public class LiteDatabase : ILiteDatabase, IDisposable { private readonly ILiteEngine _engine; private readonly BsonMapper _mapper; private readonly bool _disposeOnClose; private ILiteStorage _fs; public BsonMapper Mapper => _mapper; public ILiteStorage FileStorage => _fs ?? (_fs = GetStorage()); public int UserVersion { get { return _engine.Pragma("USER_VERSION"); } set { _engine.Pragma("USER_VERSION", value); } } public TimeSpan Timeout { get { return TimeSpan.FromSeconds(_engine.Pragma("TIMEOUT").AsInt32); } set { _engine.Pragma("TIMEOUT", (int)value.TotalSeconds); } } public bool UtcDate { get { return _engine.Pragma("UTC_DATE"); } set { _engine.Pragma("UTC_DATE", value); } } public long LimitSize { get { return _engine.Pragma("LIMIT_SIZE"); } set { _engine.Pragma("LIMIT_SIZE", value); } } public int CheckpointSize { get { return _engine.Pragma("CHECKPOINT"); } set { _engine.Pragma("CHECKPOINT", value); } } public Collation Collation => new Collation(_engine.Pragma("COLLATION").AsString); public LiteDatabase(string connectionString, BsonMapper mapper = null) : this(new ConnectionString(connectionString), mapper) { } public LiteDatabase(ConnectionString connectionString, BsonMapper mapper = null) { if (connectionString == null) { throw new ArgumentNullException("connectionString"); } _engine = connectionString.CreateEngine(); _mapper = mapper ?? BsonMapper.Global; _disposeOnClose = true; } public LiteDatabase(Stream stream, BsonMapper mapper = null, Stream logStream = null) { EngineSettings settings = new EngineSettings { DataStream = (stream ?? throw new ArgumentNullException("stream")), LogStream = logStream }; _engine = new LiteEngine(settings); _mapper = mapper ?? BsonMapper.Global; _disposeOnClose = true; } public LiteDatabase(ILiteEngine engine, BsonMapper mapper = null, bool disposeOnClose = true) { _engine = engine ?? throw new ArgumentNullException("engine"); _mapper = mapper ?? BsonMapper.Global; _disposeOnClose = disposeOnClose; } public ILiteCollection GetCollection(string name, BsonAutoId autoId = BsonAutoId.ObjectId) { return new LiteCollection(name, autoId, _engine, _mapper); } public ILiteCollection GetCollection() { return GetCollection(null); } public ILiteCollection GetCollection(BsonAutoId autoId) { return GetCollection(null, autoId); } public ILiteCollection GetCollection(string name, BsonAutoId autoId = BsonAutoId.ObjectId) { if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } return new LiteCollection(name, autoId, _engine, _mapper); } public bool BeginTrans() { return _engine.BeginTrans(); } public bool Commit() { return _engine.Commit(); } public bool Rollback() { return _engine.Rollback(); } public ILiteStorage GetStorage(string filesCollection = "_files", string chunksCollection = "_chunks") { return new LiteStorage(this, filesCollection, chunksCollection); } public IEnumerable GetCollectionNames() { return (from x in GetCollection("$cols").Query().Where("type = 'user'").ToDocuments() select x["name"].AsString).ToArray(); } public bool CollectionExists(string name) { if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } return GetCollectionNames().Contains(name, StringComparer.OrdinalIgnoreCase); } public bool DropCollection(string name) { if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } return _engine.DropCollection(name); } public bool RenameCollection(string oldName, string newName) { if (oldName.IsNullOrWhiteSpace()) { throw new ArgumentNullException("oldName"); } if (newName.IsNullOrWhiteSpace()) { throw new ArgumentNullException("newName"); } return _engine.RenameCollection(oldName, newName); } public IBsonDataReader Execute(TextReader commandReader, BsonDocument parameters = null) { if (commandReader == null) { throw new ArgumentNullException("commandReader"); } Tokenizer tokenizer = new Tokenizer(commandReader); return new SqlParser(_engine, tokenizer, parameters).Execute(); } public IBsonDataReader Execute(string command, BsonDocument parameters = null) { if (command == null) { throw new ArgumentNullException("command"); } Tokenizer tokenizer = new Tokenizer(command); return new SqlParser(_engine, tokenizer, parameters).Execute(); } public IBsonDataReader Execute(string command, params BsonValue[] args) { BsonDocument bsonDocument = new BsonDocument(); int num = 0; foreach (BsonValue value in args) { bsonDocument[num.ToString()] = value; num++; } return Execute(command, bsonDocument); } public void Checkpoint() { _engine.Checkpoint(); } public long Rebuild(RebuildOptions options = null) { return _engine.Rebuild(options ?? new RebuildOptions()); } public BsonValue Pragma(string name) { return _engine.Pragma(name); } public BsonValue Pragma(string name, BsonValue value) { return _engine.Pragma(name, value); } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } ~LiteDatabase() { Dispose(disposing: false); } protected virtual void Dispose(bool disposing) { if (disposing && _disposeOnClose) { _engine.Dispose(); } } } public class LiteQueryable : ILiteQueryable, ILiteQueryableResult { protected readonly ILiteEngine _engine; protected readonly BsonMapper _mapper; protected readonly string _collection; protected readonly Query _query; private readonly bool _isSimpleType = Reflection.IsSimpleType(typeof(T)); internal LiteQueryable(ILiteEngine engine, BsonMapper mapper, string collection, Query query) { _engine = engine; _mapper = mapper; _collection = collection; _query = query; } public ILiteQueryable Include(Expression> path) { _query.Includes.Add(_mapper.GetExpression(path)); return this; } public ILiteQueryable Include(BsonExpression path) { _query.Includes.Add(path); return this; } public ILiteQueryable Include(List paths) { _query.Includes.AddRange(paths); return this; } public ILiteQueryable Where(BsonExpression predicate) { _query.Where.Add(predicate); return this; } public ILiteQueryable Where(string predicate, BsonDocument parameters) { _query.Where.Add(BsonExpression.Create(predicate, parameters)); return this; } public ILiteQueryable Where(string predicate, params BsonValue[] args) { _query.Where.Add(BsonExpression.Create(predicate, args)); return this; } public ILiteQueryable Where(Expression> predicate) { return Where(_mapper.GetExpression(predicate)); } public ILiteQueryable OrderBy(BsonExpression keySelector, int order = 1) { if (_query.OrderBy != null) { throw new ArgumentException("ORDER BY already defined in this query builder"); } _query.OrderBy = keySelector; _query.Order = order; return this; } public ILiteQueryable OrderBy(Expression> keySelector, int order = 1) { return OrderBy(_mapper.GetExpression(keySelector), order); } public ILiteQueryable OrderByDescending(BsonExpression keySelector) { return OrderBy(keySelector, -1); } public ILiteQueryable OrderByDescending(Expression> keySelector) { return OrderBy(keySelector, -1); } public ILiteQueryable GroupBy(BsonExpression keySelector) { if (_query.GroupBy != null) { throw new ArgumentException("GROUP BY already defined in this query"); } _query.GroupBy = keySelector; return this; } public ILiteQueryable Having(BsonExpression predicate) { if (_query.Having != null) { throw new ArgumentException("HAVING already defined in this query"); } _query.Having = predicate; return this; } public ILiteQueryableResult Select(BsonExpression selector) { _query.Select = selector; return new LiteQueryable(_engine, _mapper, _collection, _query); } public ILiteQueryableResult Select(Expression> selector) { if (_query.GroupBy != null) { throw new ArgumentException("Use Select(BsonExpression selector) when using GroupBy query"); } _query.Select = _mapper.GetExpression(selector); return new LiteQueryable(_engine, _mapper, _collection, _query); } public ILiteQueryableResult ForUpdate() { _query.ForUpdate = true; return this; } public ILiteQueryableResult Offset(int offset) { _query.Offset = offset; return this; } public ILiteQueryableResult Skip(int offset) { return Offset(offset); } public ILiteQueryableResult Limit(int limit) { _query.Limit = limit; return this; } public IBsonDataReader ExecuteReader() { _query.ExplainPlan = false; return _engine.Query(_collection, _query); } public IEnumerable ToDocuments() { using IBsonDataReader reader = ExecuteReader(); while (reader.Read()) { yield return reader.Current as BsonDocument; } } public IEnumerable ToEnumerable() { if (_isSimpleType) { return from x in ToDocuments() select x[x.Keys.First()] into x select (T)_mapper.Deserialize(typeof(T), x); } return from x in ToDocuments() select (T)_mapper.Deserialize(typeof(T), x); } public List ToList() { return ToEnumerable().ToList(); } public T[] ToArray() { return ToEnumerable().ToArray(); } public BsonDocument GetPlan() { _query.ExplainPlan = true; return _engine.Query(_collection, _query).ToEnumerable().FirstOrDefault()?.AsDocument; } public T Single() { return ToEnumerable().Single(); } public T SingleOrDefault() { return ToEnumerable().SingleOrDefault(); } public T First() { return ToEnumerable().First(); } public T FirstOrDefault() { return ToEnumerable().FirstOrDefault(); } public int Count() { BsonExpression select = _query.Select; try { Select("{ count: COUNT(*._id) }"); return ToDocuments().Single()["count"].AsInt32; } finally { _query.Select = select; } } public long LongCount() { BsonExpression select = _query.Select; try { Select("{ count: COUNT(*._id) }"); return ToDocuments().Single()["count"].AsInt64; } finally { _query.Select = select; } } public bool Exists() { BsonExpression select = _query.Select; try { Select("{ exists: ANY(*._id) }"); return ToDocuments().Single()["exists"].AsBoolean; } finally { _query.Select = select; } } public int Into(string newCollection, BsonAutoId autoId = BsonAutoId.ObjectId) { _query.Into = newCollection; _query.IntoAutoId = autoId; using IBsonDataReader bsonDataReader = ExecuteReader(); return bsonDataReader.Current.AsInt32; } } public class LiteRepository : ILiteRepository, IDisposable { private readonly ILiteDatabase _db; public ILiteDatabase Database => _db; public LiteRepository(ILiteDatabase database) { _db = database; } public LiteRepository(string connectionString, BsonMapper mapper = null) { _db = new LiteDatabase(connectionString, mapper); } public LiteRepository(ConnectionString connectionString, BsonMapper mapper = null) { _db = new LiteDatabase(connectionString, mapper); } public LiteRepository(Stream stream, BsonMapper mapper = null, Stream logStream = null) { _db = new LiteDatabase(stream, mapper, logStream); } public BsonValue Insert(T entity, string collectionName = null) { return _db.GetCollection(collectionName).Insert(entity); } public int Insert(IEnumerable entities, string collectionName = null) { return _db.GetCollection(collectionName).Insert(entities); } public bool Update(T entity, string collectionName = null) { return _db.GetCollection(collectionName).Update(entity); } public int Update(IEnumerable entities, string collectionName = null) { return _db.GetCollection(collectionName).Update(entities); } public bool Upsert(T entity, string collectionName = null) { return _db.GetCollection(collectionName).Upsert(entity); } public int Upsert(IEnumerable entities, string collectionName = null) { return _db.GetCollection(collectionName).Upsert(entities); } public bool Delete(BsonValue id, string collectionName = null) { return _db.GetCollection(collectionName).Delete(id); } public int DeleteMany(BsonExpression predicate, string collectionName = null) { return _db.GetCollection(collectionName).DeleteMany(predicate); } public int DeleteMany(Expression> predicate, string collectionName = null) { return _db.GetCollection(collectionName).DeleteMany(predicate); } public ILiteQueryable Query(string collectionName = null) { return _db.GetCollection(collectionName).Query(); } public bool EnsureIndex(string name, BsonExpression expression, bool unique = false, string collectionName = null) { return _db.GetCollection(collectionName).EnsureIndex(name, expression, unique); } public bool EnsureIndex(BsonExpression expression, bool unique = false, string collectionName = null) { return _db.GetCollection(collectionName).EnsureIndex(expression, unique); } public bool EnsureIndex(Expression> keySelector, bool unique = false, string collectionName = null) { return _db.GetCollection(collectionName).EnsureIndex(keySelector, unique); } public bool EnsureIndex(string name, Expression> keySelector, bool unique = false, string collectionName = null) { return _db.GetCollection(collectionName).EnsureIndex(name, keySelector, unique); } public T SingleById(BsonValue id, string collectionName = null) { return _db.GetCollection(collectionName).Query().Where("_id = @0", id) .Single(); } public List Fetch(BsonExpression predicate, string collectionName = null) { return Query(collectionName).Where(predicate).ToList(); } public List Fetch(Expression> predicate, string collectionName = null) { return Query(collectionName).Where(predicate).ToList(); } public T First(BsonExpression predicate, string collectionName = null) { return Query(collectionName).Where(predicate).First(); } public T First(Expression> predicate, string collectionName = null) { return Query(collectionName).Where(predicate).First(); } public T FirstOrDefault(BsonExpression predicate, string collectionName = null) { return Query(collectionName).Where(predicate).FirstOrDefault(); } public T FirstOrDefault(Expression> predicate, string collectionName = null) { return Query(collectionName).Where(predicate).FirstOrDefault(); } public T Single(BsonExpression predicate, string collectionName = null) { return Query(collectionName).Where(predicate).Single(); } public T Single(Expression> predicate, string collectionName = null) { return Query(collectionName).Where(predicate).Single(); } public T SingleOrDefault(BsonExpression predicate, string collectionName = null) { return Query(collectionName).Where(predicate).SingleOrDefault(); } public T SingleOrDefault(Expression> predicate, string collectionName = null) { return Query(collectionName).Where(predicate).SingleOrDefault(); } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } ~LiteRepository() { Dispose(disposing: false); } protected virtual void Dispose(bool disposing) { if (disposing) { _db.Dispose(); } } } public class BsonCtorAttribute : Attribute { } public class BsonFieldAttribute : Attribute { public string Name { get; set; } public BsonFieldAttribute(string name) { Name = name; } public BsonFieldAttribute() { } } public class BsonIdAttribute : Attribute { public bool AutoId { get; private set; } public BsonIdAttribute() { AutoId = true; } public BsonIdAttribute(bool autoId) { AutoId = autoId; } } public class BsonIgnoreAttribute : Attribute { } public class BsonRefAttribute : Attribute { public string Collection { get; set; } public BsonRefAttribute(string collection) { Collection = collection; } public BsonRefAttribute() { Collection = null; } } public class BsonMapper { public delegate BsonValue DeserializationCallback(BsonMapper sender, Type target, BsonValue value); private readonly Dictionary _entities = new Dictionary(); private readonly ConcurrentDictionary> _customSerializer = new ConcurrentDictionary>(); private readonly ConcurrentDictionary> _customDeserializer = new ConcurrentDictionary>(); private readonly Func _typeInstantiator; private readonly ITypeNameBinder _typeNameBinder; public static BsonMapper Global = new BsonMapper(); public Func ResolveFieldName; public Action ResolveMember; public Func ResolveCollectionName; private readonly Regex _lowerCaseDelimiter = new Regex("(?!(^[A-Z]))([A-Z])", RegexOptions.Compiled); private readonly HashSet _bsonTypes = new HashSet { typeof(string), typeof(int), typeof(long), typeof(bool), typeof(Guid), typeof(DateTime), typeof(byte[]), typeof(ObjectId), typeof(double), typeof(decimal) }; private readonly HashSet _basicTypes = new HashSet { typeof(short), typeof(ushort), typeof(uint), typeof(float), typeof(char), typeof(byte), typeof(sbyte) }; public bool SerializeNullValues { get; set; } public bool TrimWhitespace { get; set; } public bool EmptyStringToNull { get; set; } public bool EnumAsInteger { get; set; } public bool IncludeFields { get; set; } public bool IncludeNonPublic { get; set; } public int MaxDepth { get; set; } public DeserializationCallback? OnDeserialization { get; set; } public BsonMapper(Func customTypeInstantiator = null, ITypeNameBinder typeNameBinder = null) { SerializeNullValues = false; TrimWhitespace = true; EmptyStringToNull = true; EnumAsInteger = false; ResolveFieldName = (string s) => s; ResolveMember = delegate { }; ResolveCollectionName = (Type t) => (!Reflection.IsEnumerable(t)) ? t.Name : Reflection.GetListItemType(t).Name; IncludeFields = false; MaxDepth = 20; _typeInstantiator = customTypeInstantiator ?? ((Func)((Type t) => null)); _typeNameBinder = typeNameBinder ?? DefaultTypeNameBinder.Instance; RegisterType((Uri uri) => uri.IsAbsoluteUri ? uri.AbsoluteUri : uri.ToString(), (BsonValue bson) => new Uri(bson.AsString)); RegisterType((DateTimeOffset value) => new BsonValue(value.UtcDateTime), (BsonValue bson) => bson.AsDateTime.ToUniversalTime()); RegisterType((TimeSpan value) => new BsonValue(value.Ticks), (BsonValue bson) => new TimeSpan(bson.AsInt64)); RegisterType((Regex r) => (r.Options != 0) ? new BsonDocument { { "p", r.ToString() }, { "o", (int)r.Options } } : new BsonValue(r.ToString()), (BsonValue value) => (!value.IsString) ? new Regex(value.AsDocument["p"].AsString, (RegexOptions)value.AsDocument["o"].AsInt32) : new Regex(value)); } public void RegisterType(Func serialize, Func deserialize) { _customSerializer[typeof(T)] = (object o) => serialize((T)o); _customDeserializer[typeof(T)] = (BsonValue b) => deserialize(b); } public void RegisterType(Type type, Func serialize, Func deserialize) { _customSerializer[type] = (object o) => serialize(o); _customDeserializer[type] = (BsonValue b) => deserialize(b); } public EntityBuilder Entity() { return new EntityBuilder(this, _typeNameBinder); } public BsonExpression GetExpression(Expression> predicate) { return new LinqExpressionVisitor(this, predicate).Resolve(typeof(K) == typeof(bool)); } public BsonExpression GetIndexExpression(Expression> predicate) { return new LinqExpressionVisitor(this, predicate).Resolve(predicate: false); } public BsonMapper UseCamelCase() { ResolveFieldName = (string s) => char.ToLower(s[0]) + s.Substring(1); return this; } public BsonMapper UseLowerCaseDelimiter(char delimiter = '_') { ResolveFieldName = (string s) => _lowerCaseDelimiter.Replace(s, delimiter + "$2").ToLower(); return this; } internal EntityMapper GetEntityMapper(Type type) { if (!_entities.TryGetValue(type, out var value)) { lock (_entities) { if (!_entities.TryGetValue(type, out value)) { return BuildAddEntityMapper(type); } } } return value; } protected virtual EntityMapper BuildAddEntityMapper(Type type) { EntityMapper entityMapper = new EntityMapper(type); _entities[type] = entityMapper; Type typeFromHandle = typeof(BsonIdAttribute); Type typeFromHandle2 = typeof(BsonIgnoreAttribute); Type typeFromHandle3 = typeof(BsonFieldAttribute); Type typeFromHandle4 = typeof(BsonRefAttribute); IEnumerable typeMembers = GetTypeMembers(type); MemberInfo idMember = GetIdMember(typeMembers); foreach (MemberInfo item in typeMembers) { if (!CustomAttributeExtensions.IsDefined(item, typeFromHandle2, inherit: true)) { string name = ResolveFieldName(item.Name); BsonFieldAttribute bsonFieldAttribute = (BsonFieldAttribute)CustomAttributeExtensions.GetCustomAttributes(item, typeFromHandle3, inherit: true).FirstOrDefault(); if (bsonFieldAttribute != null && bsonFieldAttribute.Name != null) { name = bsonFieldAttribute.Name; } if (item == idMember) { name = "_id"; } GenericGetter getter = Reflection.CreateGenericGetter(type, item); GenericSetter setter = Reflection.CreateGenericSetter(type, item); BsonIdAttribute bsonIdAttribute = (BsonIdAttribute)CustomAttributeExtensions.GetCustomAttributes(item, typeFromHandle, inherit: true).FirstOrDefault(); Type type2 = ((item is PropertyInfo) ? (item as PropertyInfo).PropertyType : (item as FieldInfo).FieldType); bool flag = Reflection.IsEnumerable(type2); MemberMapper memberMapper = new MemberMapper { AutoId = (bsonIdAttribute?.AutoId ?? true), FieldName = name, MemberName = item.Name, DataType = type2, IsEnumerable = flag, UnderlyingType = (flag ? Reflection.GetListItemType(type2) : type2), Getter = getter, Setter = setter }; BsonRefAttribute bsonRefAttribute = (BsonRefAttribute)CustomAttributeExtensions.GetCustomAttributes(item, typeFromHandle4, inherit: false).FirstOrDefault(); if (bsonRefAttribute != null && item is PropertyInfo) { RegisterDbRef(this, memberMapper, _typeNameBinder, bsonRefAttribute.Collection ?? ResolveCollectionName((item as PropertyInfo).PropertyType)); } ResolveMember?.Invoke(type, item, memberMapper); if (memberMapper.FieldName != null && !entityMapper.Members.Any((MemberMapper x) => x.FieldName.Equals(name, StringComparison.OrdinalIgnoreCase)) && !memberMapper.IsIgnore) { entityMapper.Members.Add(memberMapper); } } } return entityMapper; } protected virtual MemberInfo GetIdMember(IEnumerable members) { return Reflection.SelectMember(members, (MemberInfo x) => CustomAttributeExtensions.IsDefined(x, typeof(BsonIdAttribute), inherit: true), (MemberInfo x) => x.Name.Equals("Id", StringComparison.OrdinalIgnoreCase), (MemberInfo x) => x.Name.Equals(x.DeclaringType.Name + "Id", StringComparison.OrdinalIgnoreCase)); } protected virtual IEnumerable GetTypeMembers(Type type) { List list = new List(); BindingFlags bindingAttr = (IncludeNonPublic ? (BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) : (BindingFlags.Instance | BindingFlags.Public)); list.AddRange((from x in type.GetProperties(bindingAttr) where x.CanRead && x.GetIndexParameters().Length == 0 select x).Select((Func)((PropertyInfo x) => x))); if (IncludeFields) { list.AddRange((from x in type.GetFields(bindingAttr) where !x.Name.EndsWith("k__BackingField") && !x.IsStatic select x).Select((Func)((FieldInfo x) => x))); } return list; } protected virtual CreateObject GetTypeCtor(EntityMapper mapper) { Type type = mapper.ForType; List list = new List(); bool flag = false; ConstructorInfo[] constructors = type.GetConstructors(); foreach (ConstructorInfo constructorInfo in constructors) { ParameterInfo[] parameters = constructorInfo.GetParameters(); if (parameters.Length == 0) { flag = true; continue; } KeyValuePair[] paramMap = new KeyValuePair[parameters.Length]; int j; for (j = 0; j < parameters.Length; j++) { ParameterInfo parameterInfo = parameters[j]; MemberMapper memberMapper = null; foreach (MemberMapper member in mapper.Members) { if (member.MemberName.ToLower() == parameterInfo.Name.ToLower() && member.DataType == parameterInfo.ParameterType) { memberMapper = member; break; } } if (memberMapper == null) { break; } paramMap[j] = new KeyValuePair(memberMapper.FieldName, memberMapper.DataType); } if (j < parameters.Length) { continue; } CreateObject createObject = (BsonDocument value) => Activator.CreateInstance(type, paramMap.Select((KeyValuePair x) => Deserialize(x.Value, value[x.Key])).ToArray()); if (constructorInfo.GetCustomAttribute() != null) { return createObject; } list.Add(createObject); } if (flag) { return null; } return list.FirstOrDefault(); } internal static void RegisterDbRef(BsonMapper mapper, MemberMapper member, ITypeNameBinder typeNameBinder, string collection) { member.IsDbRef = true; if (member.IsEnumerable) { RegisterDbRefList(mapper, member, typeNameBinder, collection); } else { RegisterDbRefItem(mapper, member, typeNameBinder, collection); } } private static void RegisterDbRefItem(BsonMapper mapper, MemberMapper member, ITypeNameBinder typeNameBinder, string collection) { EntityMapper entity = mapper.GetEntityMapper(member.DataType); member.Serialize = delegate(object obj, BsonMapper m) { if (obj == null) { return BsonValue.Null; } object obj2 = (entity.Id ?? throw new LiteException(0, "There is no _id field mapped in your type: " + member.DataType.FullName)).Getter(obj); BsonDocument bsonDocument = new BsonDocument { ["$id"] = m.Serialize(obj2.GetType(), obj2, 0), ["$ref"] = collection }; if (member.DataType != obj.GetType()) { bsonDocument["$type"] = typeNameBinder.GetName(obj.GetType()); } return bsonDocument; }; member.Deserialize = delegate(BsonValue bson, BsonMapper m) { if (bson == null || !bson.IsDocument) { return null; } BsonDocument asDocument = bson.AsDocument; BsonValue value = asDocument["$id"]; bool num = asDocument["$missing"] == true; bool flag = !asDocument.ContainsKey("$ref"); if (num) { return null; } if (flag) { asDocument["_id"] = value; if (asDocument.ContainsKey("$type")) { asDocument["_type"] = bson["$type"]; } return m.Deserialize(entity.ForType, asDocument); } return m.Deserialize(entity.ForType, asDocument.ContainsKey("$type") ? new BsonDocument { ["_id"] = value, ["_type"] = bson["$type"] } : new BsonDocument { ["_id"] = value }); }; } private static void RegisterDbRefList(BsonMapper mapper, MemberMapper member, ITypeNameBinder typeNameBinder, string collection) { EntityMapper entity = mapper.GetEntityMapper(member.UnderlyingType); member.Serialize = delegate(object list, BsonMapper m) { if (list == null) { return BsonValue.Null; } BsonArray bsonArray2 = new BsonArray(); MemberMapper id = entity.Id; foreach (object item in (IEnumerable)list) { if (item != null) { object obj = id.Getter(item); BsonDocument bsonDocument2 = new BsonDocument { ["$id"] = m.Serialize(obj.GetType(), obj, 0), ["$ref"] = collection }; if (member.UnderlyingType != item.GetType()) { bsonDocument2["$type"] = typeNameBinder.GetName(item.GetType()); } bsonArray2.Add(bsonDocument2); } } return bsonArray2; }; member.Deserialize = delegate(BsonValue bson, BsonMapper m) { if (!bson.IsArray) { return null; } BsonArray asArray = bson.AsArray; if (asArray.Count == 0) { return m.Deserialize(member.DataType, asArray); } BsonArray bsonArray = new BsonArray(); foreach (BsonValue item2 in asArray) { if (item2.IsDocument) { BsonDocument asDocument = item2.AsDocument; BsonValue value = asDocument["$id"]; bool flag = asDocument["$missing"] == true; bool flag2 = !asDocument.ContainsKey("$ref"); if (!flag) { if (flag2) { item2["_id"] = value; if (item2.AsDocument.ContainsKey("$type")) { item2["_type"] = item2["$type"]; } bsonArray.Add(item2); } else { BsonDocument bsonDocument = new BsonDocument { ["_id"] = value }; if (item2.AsDocument.ContainsKey("$type")) { bsonDocument["_type"] = item2["$type"]; } bsonArray.Add(bsonDocument); } } } } return m.Deserialize(member.DataType, bsonArray); }; } public virtual object ToObject(Type type, BsonDocument doc) { if (doc == null) { throw new ArgumentNullException("doc"); } if (type == typeof(BsonDocument)) { return doc; } return Deserialize(type, doc); } public virtual T ToObject(BsonDocument doc) { return (T)ToObject(typeof(T), doc); } public T Deserialize(BsonValue value) { if (value == null) { return default(T); } return (T)Deserialize(typeof(T), value); } public object Deserialize(Type type, BsonValue value) { if (OnDeserialization != null) { BsonValue bsonValue = OnDeserialization(this, type, value); if ((object)bsonValue != null) { value = bsonValue; } } if (value.IsNull) { return null; } if (Reflection.IsNullable(type)) { type = Reflection.UnderlyingTypeOf(type); } if (_customDeserializer.TryGetValue(type, out var value2)) { return value2(value); } TypeInfo typeInfo = type.GetTypeInfo(); if (type == typeof(BsonValue)) { return value; } if (type == typeof(BsonDocument)) { return value.AsDocument; } if (type == typeof(BsonArray)) { return value.AsArray; } if (_bsonTypes.Contains(type)) { return value.RawValue; } if (_basicTypes.Contains(type)) { return Convert.ChangeType(value.RawValue, type); } if (type == typeof(ulong)) { return (ulong)value.AsInt64; } if (typeInfo.IsEnum) { if (value.IsString) { return Enum.Parse(type, value.AsString); } if (value.IsNumber) { return Enum.ToObject(type, value.AsInt32); } } else { if (value.IsArray) { if (type == typeof(object)) { return DeserializeArray(typeof(object), value.AsArray); } if (type.IsArray) { return DeserializeArray(type.GetElementType(), value.AsArray); } return DeserializeList(type, value.AsArray); } if (value.IsDocument) { if (type.IsAnonymousType()) { return DeserializeAnonymousType(type, value.AsDocument); } BsonDocument asDocument = value.AsDocument; if (asDocument.TryGetValue("_type", out var value3) && value3.IsString) { Type type2 = _typeNameBinder.GetType(value3.AsString); if (type2 == null) { throw LiteException.InvalidTypedName(value3.AsString); } if (!type.IsAssignableFrom(type2)) { throw LiteException.DataTypeNotAssignable(type.FullName, type2.FullName); } type = type2; } else if (type == typeof(object)) { type = typeof(Dictionary); } EntityMapper entity = GetEntityMapper(type); if (entity.CreateInstance == null) { entity.CreateInstance = GetTypeCtor(entity) ?? ((CreateObject)((BsonDocument v) => Reflection.CreateInstance(entity.ForType))); } object obj = _typeInstantiator(type) ?? entity.CreateInstance(asDocument); if (obj is IDictionary dict) { if (obj.GetType().GetTypeInfo().IsGenericType) { Type k = type.GetGenericArguments()[0]; Type t = type.GetGenericArguments()[1]; DeserializeDictionary(k, t, dict, value.AsDocument); } else { DeserializeDictionary(typeof(object), typeof(object), dict, value.AsDocument); } } else { DeserializeObject(entity, obj, asDocument); } return obj; } } return value.RawValue; } private object DeserializeArray(Type type, BsonArray array) { Array array2 = Array.CreateInstance(type, array.Count); int num = 0; foreach (BsonValue item in array) { array2.SetValue(Deserialize(type, item), num++); } return array2; } private object DeserializeList(Type type, BsonArray value) { Type listItemType = Reflection.GetListItemType(type); IEnumerable enumerable = (IEnumerable)Reflection.CreateInstance(type); if (enumerable is IList list) { foreach (BsonValue item in value) { list.Add(Deserialize(listItemType, item)); } } else { MethodInfo method = type.GetMethod("Add", new Type[1] { listItemType }); foreach (BsonValue item2 in value) { method.Invoke(enumerable, new object[1] { Deserialize(listItemType, item2) }); } } return enumerable; } private void DeserializeDictionary(Type K, Type T, IDictionary dict, BsonDocument value) { bool isEnum = K.GetTypeInfo().IsEnum; foreach (KeyValuePair element in value.GetElements()) { object key = (isEnum ? Enum.Parse(K, element.Key) : ((K == typeof(Uri)) ? new Uri(element.Key) : Convert.ChangeType(element.Key, K))); object value2 = Deserialize(T, element.Value); dict.Add(key, value2); } } private void DeserializeObject(EntityMapper entity, object obj, BsonDocument value) { foreach (MemberMapper item in entity.Members.Where((MemberMapper x) => x.Setter != null)) { if (value.TryGetValue(item.FieldName, out var value2)) { if (item.Deserialize != null) { item.Setter(obj, item.Deserialize(value2, this)); } else { item.Setter(obj, Deserialize(item.DataType, value2)); } } } } private object DeserializeAnonymousType(Type type, BsonDocument value) { List list = new List(); ParameterInfo[] parameters = type.GetConstructors()[0].GetParameters(); foreach (ParameterInfo parameterInfo in parameters) { object obj = Deserialize(parameterInfo.ParameterType, value[parameterInfo.Name]); if (obj == null && StringComparer.OrdinalIgnoreCase.Equals(parameterInfo.Name, "Id") && value.TryGetValue("_id", out var value2)) { obj = Deserialize(parameterInfo.ParameterType, value2); } list.Add(obj); } return Activator.CreateInstance(type, list.ToArray()); } public virtual BsonDocument ToDocument(Type type, object entity) { if (entity == null) { throw new ArgumentNullException("entity"); } if (entity is BsonDocument) { return (BsonDocument)entity; } return Serialize(type, entity, 0).AsDocument; } public virtual BsonDocument ToDocument(T entity) { return ToDocument(typeof(T), entity)?.AsDocument; } public BsonValue Serialize(T obj) { return Serialize(typeof(T), obj, 0); } public BsonValue Serialize(Type type, object obj) { return Serialize(type, obj, 0); } internal BsonValue Serialize(Type type, object obj, int depth) { if (++depth > MaxDepth) { throw LiteException.DocumentMaxDepth(MaxDepth, type); } if (obj == null) { return BsonValue.Null; } if (obj is BsonValue result) { return result; } if (_customSerializer.TryGetValue(type, out var value) || _customSerializer.TryGetValue(obj.GetType(), out value)) { return value(obj); } if (obj is string) { string text = (TrimWhitespace ? (obj as string).Trim() : ((string)obj)); if (EmptyStringToNull && text.Length == 0) { return BsonValue.Null; } return new BsonValue(text); } if (obj is int) { return new BsonValue((int)obj); } if (obj is long) { return new BsonValue((long)obj); } if (obj is double) { return new BsonValue((double)obj); } if (obj is decimal) { return new BsonValue((decimal)obj); } if (obj is byte[]) { return new BsonValue((byte[])obj); } if (obj is ObjectId) { return new BsonValue((ObjectId)obj); } if (obj is Guid) { return new BsonValue((Guid)obj); } if (obj is bool) { return new BsonValue((bool)obj); } if (obj is DateTime) { return new BsonValue((DateTime)obj); } if (obj is short || obj is ushort || obj is byte || obj is sbyte) { return new BsonValue(Convert.ToInt32(obj)); } if (obj is uint) { return new BsonValue(Convert.ToInt64(obj)); } if (obj is ulong) { return new BsonValue((long)(ulong)obj); } if (obj is float) { return new BsonValue(Convert.ToDouble(obj)); } if (obj is char) { return new BsonValue(obj.ToString()); } if (obj is Enum) { if (EnumAsInteger) { return new BsonValue((int)obj); } return new BsonValue(obj.ToString()); } if (obj is IDictionary dict) { if (type == typeof(object)) { type = obj.GetType(); } Type type2 = (type.GetTypeInfo().IsGenericType ? type.GetGenericArguments()[1] : typeof(object)); return SerializeDictionary(type2, dict, depth); } if (obj is IEnumerable) { return SerializeArray(Reflection.GetListItemType(type), obj as IEnumerable, depth); } return SerializeObject(type, obj, depth); } private BsonArray SerializeArray(Type type, IEnumerable array, int depth) { BsonArray bsonArray = new BsonArray(); foreach (object item in array) { bsonArray.Add(Serialize(type, item, depth)); } return bsonArray; } private BsonDocument SerializeDictionary(Type type, IDictionary dict, int depth) { BsonDocument bsonDocument = new BsonDocument(); foreach (object key in dict.Keys) { object obj = dict[key]; string name = key.ToString(); if (key is DateTime dateTime) { name = dateTime.ToString("o"); } bsonDocument[name] = Serialize(type, obj, depth); } return bsonDocument; } private BsonDocument SerializeObject(Type type, object obj, int depth) { Type type2 = obj.GetType(); BsonDocument bsonDocument = new BsonDocument(); EntityMapper entityMapper = GetEntityMapper(type2); if (type != type2) { bsonDocument["_type"] = new BsonValue(_typeNameBinder.GetName(type2)); } foreach (MemberMapper item in entityMapper.Members.Where((MemberMapper x) => x.Getter != null)) { object obj2 = item.Getter(obj); if (obj2 != null || SerializeNullValues || !(item.FieldName != "_id")) { if (item.Serialize != null) { bsonDocument[item.FieldName] = item.Serialize(obj2, this); } else { bsonDocument[item.FieldName] = Serialize(item.DataType, obj2, depth); } } } return bsonDocument; } } public class EntityBuilder { private readonly BsonMapper _mapper; private readonly EntityMapper _entity; private readonly ITypeNameBinder _typeNameBinder; internal EntityBuilder(BsonMapper mapper, ITypeNameBinder typeNameBinder) { _mapper = mapper; _typeNameBinder = typeNameBinder; _entity = mapper.GetEntityMapper(typeof(T)); } public EntityBuilder Ignore(Expression> member) { return GetMember(member, delegate(MemberMapper p) { _entity.Members.Remove(p); }); } public EntityBuilder Field(Expression> member, string field) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return GetMember(member, delegate(MemberMapper p) { p.FieldName = field; }); } public EntityBuilder Id(Expression> member, bool autoId = true) { return GetMember(member, delegate(MemberMapper p) { MemberMapper memberMapper = _entity.Members.FirstOrDefault((MemberMapper x) => x.FieldName == "_id"); if (memberMapper != null) { memberMapper.FieldName = _mapper.ResolveFieldName(memberMapper.MemberName); memberMapper.AutoId = false; } p.FieldName = "_id"; p.AutoId = autoId; }); } public EntityBuilder Ctor(Func createInstance) { _entity.CreateInstance = (BsonDocument v) => createInstance(v); return this; } public EntityBuilder DbRef(Expression> member, string collection = null) { return GetMember(member, delegate(MemberMapper p) { BsonMapper.RegisterDbRef(_mapper, p, _typeNameBinder, collection ?? _mapper.ResolveCollectionName(typeof(K))); }); } private EntityBuilder GetMember(Expression> member, Action action) { if (member == null) { throw new ArgumentNullException("member"); } MemberMapper member2 = _entity.GetMember(member); if (member2 == null) { throw new ArgumentNullException("Member '" + member.GetPath() + "' not found in type '" + _entity.ForType.Name + "' (use IncludeFields in BsonMapper)"); } action(member2); return this; } } public class EntityMapper { public Type ForType { get; } public List Members { get; } = new List(); public MemberMapper Id => Members.SingleOrDefault((MemberMapper x) => x.FieldName == "_id"); public CreateObject CreateInstance { get; set; } public EntityMapper(Type forType) { ForType = forType; } public MemberMapper GetMember(Expression expr) { return Members.FirstOrDefault((MemberMapper x) => x.MemberName == expr.GetPath()); } } internal class LinqExpressionVisitor : ExpressionVisitor { private static readonly Dictionary _resolver = new Dictionary { [typeof(BsonValue)] = new BsonValueResolver(), [typeof(BsonArray)] = new BsonValueResolver(), [typeof(BsonDocument)] = new BsonValueResolver(), [typeof(Convert)] = new ConvertResolver(), [typeof(DateTime)] = new DateTimeResolver(), [typeof(int)] = new NumberResolver("INT32"), [typeof(long)] = new NumberResolver("INT64"), [typeof(decimal)] = new NumberResolver("DECIMAL"), [typeof(double)] = new NumberResolver("DOUBLE"), [typeof(ICollection)] = new ICollectionResolver(), [typeof(Enumerable)] = new EnumerableResolver(), [typeof(Guid)] = new GuidResolver(), [typeof(Math)] = new MathResolver(), [typeof(Regex)] = new RegexResolver(), [typeof(ObjectId)] = new ObjectIdResolver(), [typeof(string)] = new StringResolver(), [typeof(Nullable)] = new NullableResolver() }; private readonly BsonMapper _mapper; private readonly Expression _expr; private readonly ParameterExpression _rootParameter; private readonly BsonDocument _parameters = new BsonDocument(); private int _paramIndex; private Type _dbRefType; private readonly StringBuilder _builder = new StringBuilder(); private readonly Stack _nodes = new Stack(); public LinqExpressionVisitor(BsonMapper mapper, Expression expr) { _mapper = mapper; _expr = expr; if (expr is LambdaExpression lambdaExpression) { _rootParameter = lambdaExpression.Parameters.First(); return; } throw new NotSupportedException("Expression " + expr.ToString() + " must be a lambda expression"); } public BsonExpression Resolve(bool predicate) { Visit(_expr); Constants.ENSURE(_nodes.Count == 0, "node stack must be empty when finish expression resolve"); string text = _builder.ToString(); try { BsonExpression bsonExpression = BsonExpression.Create(text, _parameters); if (predicate && (bsonExpression.Type == BsonExpressionType.Path || bsonExpression.Type == BsonExpressionType.Call || bsonExpression.Type == BsonExpressionType.Parameter)) { text = "(" + text + " = true)"; bsonExpression = BsonExpression.Create(text, _parameters); } return bsonExpression; } catch (Exception innerException) { throw new NotSupportedException("Invalid BsonExpression when converted from Linq expression: " + _expr.ToString() + " - `" + text + "`", innerException); } } protected override Expression VisitLambda(Expression node) { Expression result = base.VisitLambda(node); _builder.Length--; return result; } protected override Expression VisitInvocation(InvocationExpression node) { Expression result = base.VisitInvocation(node); _builder.Length--; return result; } protected override Expression VisitParameter(ParameterExpression node) { _builder.Append(_rootParameter.Equals(node) ? "$" : "@"); return base.VisitParameter(node); } protected override Expression VisitMember(MemberExpression node) { bool flag = ParameterExpressionVisitor.Test(node); MemberInfo member = node.Member; if (TryGetResolver(member.DeclaringType, out var typeResolver)) { string text = typeResolver.ResolveMember(member); if (text == null) { throw new NotSupportedException("Member " + member.Name + " are not support in " + member.DeclaringType.Name + " when convert to BsonExpression (" + node.ToString() + ")."); } ResolvePattern(text, node.Expression, new Expression[0]); } else if (node.Expression != null) { _nodes.Push(node); base.Visit(node.Expression); if (flag) { string value = ResolveMember(member); _builder.Append(value); } } else { object value2 = Evaluate(node); base.Visit(Expression.Constant(value2)); } if (_nodes.Count > 0) { _nodes.Pop(); } return node; } protected override Expression VisitMethodCall(MethodCallExpression node) { if (IsMethodIndexEval(node, out var obj, out var idx)) { Visit(obj); object obj2 = Evaluate(idx, typeof(string), typeof(int)); if (obj2 is string) { _builder.Append("."); _builder.Append($"['{obj2}']"); } else { _builder.Append($"[{obj2}]"); } return node; } if (!TryGetResolver(node.Method.DeclaringType, out var typeResolver)) { if (ParameterExpressionVisitor.Test(node)) { throw new NotSupportedException("Method " + node.Method.Name + " not available to convert to BsonExpression (" + node.ToString() + ")."); } object value = Evaluate(node); base.Visit(Expression.Constant(value)); return node; } string text = typeResolver.ResolveMethod(node.Method); if (text == null) { throw new NotSupportedException("Method " + Reflection.MethodName(node.Method) + " in " + node.Method.DeclaringType.Name + " are not supported when convert to BsonExpression (" + node.ToString() + ")."); } ResolvePattern(text, node.Object, node.Arguments); return node; } protected override Expression VisitConstant(ConstantExpression node) { object value = node.Value; while (_nodes.Count > 0 && _nodes.Peek() is MemberExpression memberExpression) { if (memberExpression.Member is FieldInfo fieldInfo) { value = fieldInfo.GetValue(value); } else if (memberExpression.Member is PropertyInfo propertyInfo) { value = propertyInfo.GetValue(value); } _nodes.Pop(); } Constants.ENSURE(_nodes.Count == 0, "counter stack must be zero to eval all properties/field over object"); string text = "p" + _paramIndex++; _builder.AppendFormat("@" + text); Type type = value?.GetType(); BsonValue value2 = ((type == null) ? BsonValue.Null : ((type == typeof(string)) ? new BsonValue((string)value) : _mapper.Serialize(value.GetType(), value))); _parameters[text] = value2; return node; } protected override Expression VisitUnary(UnaryExpression node) { if (node.NodeType == ExpressionType.Not) { if (node.Operand.NodeType == ExpressionType.MemberAccess) { _builder.Append("("); Visit(node.Operand); _builder.Append(" = false)"); } else { _builder.Append("("); Visit(node.Operand); _builder.Append(")"); _builder.Append(" = false"); } } else if (node.NodeType == ExpressionType.Convert) { Type fromType = node.Operand.Type; Type type = node.Type; if ((fromType == typeof(double) || fromType == typeof(decimal)) && (type == typeof(int) || type == typeof(long))) { string methodName = "To" + type.Name.ToString(); MethodInfo methodInfo = (from x in typeof(Convert).GetMethods() where x.Name == methodName where x.GetParameters().Length == 1 && x.GetParameters().Any((ParameterInfo z) => z.ParameterType == fromType) select x).FirstOrDefault(); if (methodInfo == null) { throw new NotSupportedException("Cast from " + fromType.Name + " are not supported when convert to BsonExpression"); } MethodCallExpression node2 = Expression.Call(null, methodInfo, node.Operand); VisitMethodCall(node2); } else { base.VisitUnary(node); } } else if (node.NodeType == ExpressionType.ArrayLength) { _builder.Append("LENGTH("); Visit(node.Operand); _builder.Append(")"); } else { base.VisitUnary(node); } return node; } protected override Expression VisitNew(NewExpression node) { if (node.Members == null) { if (!TryGetResolver(node.Type, out var typeResolver)) { throw new NotSupportedException($"New instance are not supported for {node.Type} when convert to BsonExpression ({node.ToString()})."); } string text = typeResolver.ResolveCtor(node.Constructor); if (text == null) { throw new NotSupportedException("Constructor for " + node.Type.Name + " are not supported when convert to BsonExpression (" + node.ToString() + ")."); } ResolvePattern(text, null, node.Arguments); } else { _builder.Append("{ "); for (int i = 0; i < node.Members.Count; i++) { MemberInfo memberInfo = node.Members[i]; _builder.Append((i > 0) ? ", " : ""); _builder.AppendFormat("'{0}': ", memberInfo.Name); Visit(node.Arguments[i]); } _builder.Append(" }"); } return node; } protected override Expression VisitMemberInit(MemberInitExpression node) { if (node.NewExpression.Constructor.GetParameters().Length != 0) { throw new NotSupportedException($"New instance of {node.Type} are not supported because contains ctor with parameter. Try use only property initializers: `new {node.Type.Name} {{ PropA = 1, PropB == \"John\" }}`."); } _builder.Append("{"); for (int i = 0; i < node.Bindings.Count; i++) { MemberAssignment memberAssignment = node.Bindings[i] as MemberAssignment; string text = ResolveMember(memberAssignment.Member); _builder.Append((i > 0) ? ", " : ""); _builder.Append(text.Substring(1)); _builder.Append(":"); Visit(memberAssignment.Expression); } _builder.Append("}"); return node; } protected override Expression VisitNewArray(NewArrayExpression node) { _builder.Append("[ "); for (int i = 0; i < node.Expressions.Count; i++) { _builder.Append((i > 0) ? ", " : ""); Visit(node.Expressions[i]); } _builder.Append(" ]"); return node; } protected override Expression VisitBinary(BinaryExpression node) { bool ensurePredicate = node.NodeType == ExpressionType.AndAlso || node.NodeType == ExpressionType.OrElse; if (node.NodeType == ExpressionType.Coalesce) { return VisitCoalesce(node); } if (node.NodeType == ExpressionType.ArrayIndex) { return VisitArrayIndex(node); } string @operator = GetOperator(node.NodeType); _builder.Append("("); VisitAsPredicate(node.Left, ensurePredicate); _builder.Append(@operator); if (!_mapper.EnumAsInteger && node.Left.NodeType == ExpressionType.Convert && node.Left is UnaryExpression unaryExpression && unaryExpression.Operand.Type.GetTypeInfo().IsEnum && unaryExpression.Type == typeof(int)) { VisitAsPredicate(Expression.Constant(Enum.GetName(unaryExpression.Operand.Type, Evaluate(node.Right))), ensurePredicate); } else { VisitAsPredicate(node.Right, ensurePredicate); } _builder.Append(")"); return node; } protected override Expression VisitConditional(ConditionalExpression node) { _builder.Append("IIF("); Visit(node.Test); _builder.Append(", "); Visit(node.IfTrue); _builder.Append(", "); Visit(node.IfFalse); _builder.Append(")"); return node; } private Expression VisitCoalesce(BinaryExpression node) { _builder.Append("COALESCE("); Visit(node.Left); _builder.Append(", "); Visit(node.Right); _builder.Append(")"); return node; } private Expression VisitArrayIndex(BinaryExpression node) { Visit(node.Left); _builder.Append("["); object value = Evaluate(node.Right, typeof(int)); _builder.Append(value); _builder.Append("]"); return node; } private void ResolvePattern(string pattern, Expression obj, IList args) { Tokenizer tokenizer = new Tokenizer(pattern); while (!tokenizer.EOF) { Token token = tokenizer.ReadToken(eatWhitespace: false); if (token.Type == TokenType.Hashtag) { Visit(obj); } else if (token.Type == TokenType.At && tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Int) { int index = Convert.ToInt32(tokenizer.ReadToken(eatWhitespace: false).Expect(TokenType.Int).Value); Visit(args[index]); } else if (token.Type == TokenType.Percent) { VisitEnumerablePredicate(args[1] as LambdaExpression); } else { _builder.Append((token.Type == TokenType.String) ? ("'" + token.Value + "'") : token.Value); } } } private void VisitEnumerablePredicate(LambdaExpression lambda) { Expression body = lambda.Body; if (body is BinaryExpression binaryExpression) { if (binaryExpression.Left.NodeType != ExpressionType.Parameter) { throw new LiteException(0, "Any/All requires simple parameter on left side. Eg: `x => x.Phones.Select(p => p.Number).Any(n => n > 5)`"); } string @operator = GetOperator(binaryExpression.NodeType); _builder.Append(@operator); VisitAsPredicate(binaryExpression.Right, ensurePredicate: false); return; } if (body is MethodCallExpression methodCallExpression) { if (methodCallExpression.Object.NodeType != ExpressionType.Parameter) { throw new NotSupportedException("Any/All requires simple parameter on left side. Eg: `x.Customers.Select(c => c.Name).Any(n => n.StartsWith('J'))`"); } if (!TryGetResolver(methodCallExpression.Method.DeclaringType, out var typeResolver)) { throw new NotSupportedException("Method " + methodCallExpression.Method.Name + " not available to convert to BsonExpression inside Any/All call."); } string text = typeResolver.ResolveMethod(methodCallExpression.Method); if (text == null || !text.StartsWith("#")) { throw new NotSupportedException("Method " + methodCallExpression.Method.Name + " not available to convert to BsonExpression inside Any/All call."); } ResolvePattern(text.Substring(1), methodCallExpression.Object, methodCallExpression.Arguments); return; } throw new LiteException(0, "When using Any/All method test do only simple predicate variable. Eg: `x => x.Phones.Select(p => p.Number).Any(n => n > 5)`"); } private string GetOperator(ExpressionType nodeType) { return nodeType switch { ExpressionType.Add => " + ", ExpressionType.Multiply => " * ", ExpressionType.Subtract => " - ", ExpressionType.Divide => " / ", ExpressionType.Equal => " = ", ExpressionType.NotEqual => " != ", ExpressionType.GreaterThan => " > ", ExpressionType.GreaterThanOrEqual => " >= ", ExpressionType.LessThan => " < ", ExpressionType.LessThanOrEqual => " <= ", ExpressionType.And => " AND ", ExpressionType.AndAlso => " AND ", ExpressionType.Or => " OR ", ExpressionType.OrElse => " OR ", _ => throw new NotSupportedException($"Operator not supported {nodeType}"), }; } private string ResolveMember(MemberInfo member) { string name = member.Name; bool flag = _dbRefType != null && member.DeclaringType.IsAssignableFrom(_dbRefType); MemberMapper memberMapper = _mapper.GetEntityMapper(member.DeclaringType).Members.FirstOrDefault((MemberMapper x) => x.MemberName == name); if (memberMapper == null) { throw new NotSupportedException($"Member {name} not found on BsonMapper for type {member.DeclaringType}."); } _dbRefType = (memberMapper.IsDbRef ? memberMapper.UnderlyingType : null); return "." + ((flag && memberMapper.FieldName == "_id") ? "$id" : memberMapper.FieldName); } private bool IsMethodIndexEval(MethodCallExpression node, out Expression obj, out Expression idx) { MethodInfo method = node.Method; _ = method.DeclaringType; ParameterInfo[] parameters = method.GetParameters(); if (method.Name == "get_Item" && parameters.Length == 1 && (parameters[0].ParameterType == typeof(int) || parameters[0].ParameterType == typeof(string))) { obj = node.Object; idx = node.Arguments[0]; return true; } obj = null; idx = null; return false; } private void VisitAsPredicate(Expression expr, bool ensurePredicate) { ensurePredicate = ensurePredicate && (expr.NodeType == ExpressionType.MemberAccess || expr.NodeType == ExpressionType.Call || expr.NodeType == ExpressionType.Invoke || expr.NodeType == ExpressionType.Constant); if (ensurePredicate) { _builder.Append("("); _builder.Append("("); base.Visit(expr); _builder.Append(")"); _builder.Append(" = true)"); } else { base.Visit(expr); } } private object Evaluate(Expression expr, params Type[] validTypes) { object value = null; if (expr.NodeType == ExpressionType.Constant) { ConstantExpression constantExpression = (ConstantExpression)expr; value = constantExpression.Value; } else { Delegate @delegate = Expression.Lambda(expr).Compile(); value = @delegate.DynamicInvoke(); } if (validTypes.Length != 0 && value == null) { throw new NotSupportedException($"Expression {expr} can't return null value"); } if (validTypes.Length != 0 && !validTypes.Any((Type x) => x == value.GetType())) { throw new NotSupportedException(string.Format("Expression {0} must return on of this types: {1}", expr, string.Join(", ", validTypes.Select((Type x) => "`" + x.Name + "`")))); } return value; } private bool TryGetResolver(Type declaringType, out ITypeResolver typeResolver) { bool num = Reflection.IsCollection(declaringType); bool flag = Reflection.IsEnumerable(declaringType); bool flag2 = Reflection.IsNullable(declaringType); Type key = (num ? typeof(ICollection) : (flag ? typeof(Enumerable) : (flag2 ? typeof(Nullable) : declaringType))); return _resolver.TryGetValue(key, out typeResolver); } } internal class ParameterExpressionVisitor : ExpressionVisitor { public bool IsParameter { get; private set; } protected override Expression VisitParameter(ParameterExpression node) { IsParameter = true; return base.VisitParameter(node); } public static bool Test(Expression node) { ParameterExpressionVisitor parameterExpressionVisitor = new ParameterExpressionVisitor(); parameterExpressionVisitor.Visit(node); return parameterExpressionVisitor.IsParameter; } } internal class BsonValueResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { return null; } public string ResolveMember(MemberInfo member) { switch (member.Name) { case "AsInt32": case "AsInt64": case "AsArray": case "AsDateTime": case "AsDocument": case "AsObjectId": case "AsString": case "AsBinary": case "AsDouble": case "AsBoolean": case "AsDecimal": case "AsGuid": return "#"; case "IsNull": return "IS_NULL(#)"; case "IsArray": return "IS_ARRAY(#)"; case "IsDocument": return "IS_DOCUMENT(#)"; case "IsInt32": return "IS_INT32(#)"; case "IsInt64": return "IS_INT64(#)"; case "IsDouble": return "IS_DOUBLE(#)"; case "IsDecimal": return "IS_DECIMAL(#)"; case "IsNumber": return "IS_NUMBER(#)"; case "IsBinary": return "IS_BINARY(#)"; case "IsBoolean": return "IS_BOOLEAN(#)"; case "IsString": return "IS_STRING(#)"; case "IsObjectId": return "IS_OBJECTID(#)"; case "IsGuid": return "IS_GUID(#)"; case "IsDateTime": return "IS_DATETIME(#)"; case "IsMinValue": return "IS_MINVALUE(#)"; case "IsMaxValue": return "IS_MAXVALUE(#)"; default: return null; } } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class ConvertResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { return method.Name switch { "ToInt32" => "INT32(@0)", "ToInt64" => "INT64(@0)", "ToDouble" => "DOUBLE(@0)", "ToDecimal" => "DECIMAL(@0)", "ToDateTime" => "DATE(@0)", "FromBase64String" => "BINARY(@0)", "ToBoolean" => "BOOL(@0)", "ToString" => "STRING(@0)", _ => null, }; } public string ResolveMember(MemberInfo member) { return null; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class DateTimeResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { switch (method.Name) { case "AddYears": return "DATEADD('y', @0, #)"; case "AddMonths": return "DATEADD('M', @0, #)"; case "AddDays": return "DATEADD('d', @0, #)"; case "AddHours": return "DATEADD('h', @0, #)"; case "AddMinutes": return "DATEADD('m', @0, #)"; case "AddSeconds": return "DATEADD('s', @0, #)"; case "ToString": { ParameterInfo[] parameters = method.GetParameters(); if (parameters.Length == 0) { return "STRING(#)"; } if (parameters.Length == 1 && parameters[0].ParameterType == typeof(string)) { return "FORMAT(#, @0)"; } break; } case "ToUniversalTime": return "TO_UTC(#)"; case "Parse": return "DATETIME(@0)"; case "Equals": return "# = @0"; } return null; } public string ResolveMember(MemberInfo member) { return member.Name switch { "Now" => "NOW()", "UtcNow" => "NOW_UTC()", "Today" => "TODAY()", "Year" => "YEAR(#)", "Month" => "MONTH(#)", "Day" => "DAY(#)", "Hour" => "HOUR(#)", "Minute" => "MINUTE(#)", "Second" => "SECOND(#)", "Date" => "DATETIME(YEAR(#), MONTH(#), DAY(#))", "ToLocalTime" => "TO_LOCAL(#)", "ToUniversalTime" => "TO_UTC(#)", _ => null, }; } public string ResolveCtor(ConstructorInfo ctor) { ParameterInfo[] parameters = ctor.GetParameters(); if (parameters.Length == 3 && parameters[0].ParameterType == typeof(int) && parameters[1].ParameterType == typeof(int) && parameters[2].ParameterType == typeof(int)) { return "DATETIME(@0, @1, @2)"; } return null; } } internal class EnumerableResolver : ITypeResolver { public virtual string ResolveMethod(MethodInfo method) { switch (Reflection.MethodName(method, 1)) { case "AsEnumerable()": return "@0[*]"; case "get_Item(int)": return "#[@0]"; case "ElementAt(int)": return "@0[@1]"; case "Single()": case "First()": case "SingleOrDefault()": case "FirstOrDefault()": return "@0[0]"; case "Last()": case "LastOrDefault()": return "@0[-1]"; case "Single(Func)": case "First(Func)": case "SingleOrDefault(Func)": case "FirstOrDefault(Func)": return "FIRST(FILTER(@0 => @1))"; case "Last(Func)": case "LastOrDefault(Func)": return "LAST(FILTER(@0 => @1))"; case "Where(Func)": return "FILTER(@0 => @1)"; case "Select(Func)": return "MAP(@0 => @1)"; case "Count()": return "COUNT(@0)"; case "Sum()": return "SUM(@0)"; case "Average()": return "AVG(@0)"; case "Max()": return "MAX(@0)"; case "Min()": return "MIN(@0)"; case "Count(Func)": return "COUNT(FILTER(@0 => @1))"; case "Sum(Func)": return "SUM(MAP(@0 => @1))"; case "Average(Func)": return "AVG(MAP(@0 => @1))"; case "Max(Func)": return "MAX(MAP(@0 => @1))"; case "Min(Func)": return "MIN(MAP(@0 => @1))"; case "ToList()": case "ToArray()": return "ARRAY(@0)"; case "Any(Func)": return "@0 ANY %"; case "All(Func)": return "@0 ALL %"; case "Any()": return "COUNT(@0) > 0"; default: if (method.Name == "Contains") { return "@0 ANY = @1"; } return null; } } public virtual string ResolveMember(MemberInfo member) { string name = member.Name; if (!(name == "Length")) { if (name == "Count") { return "COUNT(#)"; } return null; } return "LENGTH(#)"; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class GuidResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { return method.Name switch { "ToString" => "STRING(#)", "NewGuid" => "GUID()", "Parse" => "GUID(@0)", "TryParse" => throw new NotSupportedException("There is no TryParse translate. Use Guid.Parse()"), "Equals" => "# = @0", _ => null, }; } public string ResolveMember(MemberInfo member) { if (member.Name == "Empty") { return "GUID('00000000-0000-0000-0000-000000000000')"; } return null; } public string ResolveCtor(ConstructorInfo ctor) { ParameterInfo[] parameters = ctor.GetParameters(); if (parameters.Length == 1 && parameters[0].ParameterType == typeof(string)) { return "GUID(@0)"; } return null; } } internal class ICollectionResolver : EnumerableResolver { public override string ResolveMethod(MethodInfo method) { if (method.Name == "Contains") { return "# ANY = @0"; } return base.ResolveMethod(method); } } internal interface ITypeResolver { string ResolveMethod(MethodInfo method); string ResolveMember(MemberInfo member); string ResolveCtor(ConstructorInfo ctor); } internal class MathResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { int num = method.GetParameters().Length; switch (method.Name) { case "Abs": return "ABS(@0)"; case "Pow": return "POW(@0, @1)"; case "Round": if (num != 2) { throw new ArgumentOutOfRangeException("Method Round need 2 arguments when convert to BsonExpression"); } return "ROUND(@0, @1)"; default: return null; } } public string ResolveMember(MemberInfo member) { return null; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class NullableResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { return null; } public string ResolveMember(MemberInfo member) { string name = member.Name; if (!(name == "HasValue")) { if (name == "Value") { return "#"; } return null; } return "(IS_NULL(#) = false)"; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class NumberResolver : ITypeResolver { private readonly string _parseMethod; public NumberResolver(string parseMethod) { _parseMethod = parseMethod; } public string ResolveMethod(MethodInfo method) { switch (method.Name) { case "ToString": { ParameterInfo[] parameters = method.GetParameters(); if (parameters.Length == 0) { return "STRING(#)"; } if (parameters.Length == 1 && parameters[0].ParameterType == typeof(string)) { return "FORMAT(#, @0)"; } break; } case "Parse": return _parseMethod + "(@0)"; case "Equals": return "# = @0"; } return null; } public string ResolveMember(MemberInfo member) { return null; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class ObjectIdResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { string name = method.Name; if (!(name == "ToString")) { if (name == "Equals") { return "# = @0"; } return null; } return "STRING(#)"; } public string ResolveMember(MemberInfo member) { string name = member.Name; if (!(name == "Empty")) { if (name == "CreationTime") { return "OID_CREATIONTIME(#)"; } return null; } return "OBJECTID('000000000000000000000000')"; } public string ResolveCtor(ConstructorInfo ctor) { ParameterInfo[] parameters = ctor.GetParameters(); if (parameters.Length == 1 && parameters[0].ParameterType == typeof(string)) { return "OBJECTID(@0)"; } return null; } } internal class RegexResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { string name = method.Name; if (!(name == "Split")) { if (name == "IsMatch") { return "IS_MATCH(@0, @1)"; } return null; } return "SPLIT(@0, @1, true)"; } public string ResolveMember(MemberInfo member) { return null; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } internal class StringResolver : ITypeResolver { public string ResolveMethod(MethodInfo method) { int num = method.GetParameters().Length; switch (method.Name) { case "Count": return "LENGTH(#)"; case "Trim": return "TRIM(#)"; case "TrimStart": return "LTRIM(#)"; case "TrimEnd": return "RTRIM(#)"; case "ToUpper": return "UPPER(#)"; case "ToUpperInvariant": return "UPPER(#)"; case "ToLower": return "LOWER(#)"; case "ToLowerInvariant": return "LOWER(#)"; case "Replace": return "REPLACE(#, @0, @1)"; case "PadLeft": return "LPAD(#, @0, @1)"; case "RightLeft": return "RPAD(#, @0, @1)"; case "IndexOf": if (num != 1) { return "INDEXOF(#, @0, @1)"; } return "INDEXOF(#, @0)"; case "Substring": if (num != 1) { return "SUBSTRING(#, @0, @1)"; } return "SUBSTRING(#, @0)"; case "StartsWith": return "# LIKE (@0 + '%')"; case "Contains": return "# LIKE ('%' + @0 + '%')"; case "EndsWith": return "# LIKE ('%' + @0)"; case "ToString": return "#"; case "Equals": return "# = @0"; case "IsNullOrEmpty": return "(LENGTH(@0) = 0)"; case "IsNullOrWhiteSpace": return "(LENGTH(TRIM(@0)) = 0)"; case "Format": throw new NotImplementedException(); case "Join": throw new NotImplementedException(); default: return null; } } public string ResolveMember(MemberInfo member) { string name = member.Name; if (!(name == "Length")) { if (name == "Empty") { return "''"; } return null; } return "LENGTH(#)"; } public string ResolveCtor(ConstructorInfo ctor) { return null; } } public class MemberMapper { public bool AutoId { get; set; } public string MemberName { get; set; } public Type DataType { get; set; } public string FieldName { get; set; } public GenericGetter Getter { get; set; } public GenericSetter Setter { get; set; } public Func Serialize { get; set; } public Func Deserialize { get; set; } public bool IsDbRef { get; set; } public bool IsEnumerable { get; set; } public Type UnderlyingType { get; set; } public bool IsIgnore { get; set; } } public delegate object CreateObject(BsonDocument value); public delegate void GenericSetter(object target, object value); public delegate object GenericGetter(object obj); internal class Reflection { private static readonly Dictionary _cacheCtor = new Dictionary(); public static readonly Dictionary ConvertType = new Dictionary { [typeof(DateTime)] = typeof(BsonValue).GetProperty("AsDateTime"), [typeof(decimal)] = typeof(BsonValue).GetProperty("AsDecimal"), [typeof(double)] = typeof(BsonValue).GetProperty("AsDouble"), [typeof(long)] = typeof(BsonValue).GetProperty("AsInt64"), [typeof(int)] = typeof(BsonValue).GetProperty("AsInt32"), [typeof(bool)] = typeof(BsonValue).GetProperty("AsBoolean"), [typeof(byte[])] = typeof(BsonValue).GetProperty("AsBinary"), [typeof(BsonDocument)] = typeof(BsonValue).GetProperty("AsDocument"), [typeof(BsonArray)] = typeof(BsonValue).GetProperty("AsArray"), [typeof(ObjectId)] = typeof(BsonValue).GetProperty("AsObjectId"), [typeof(string)] = typeof(BsonValue).GetProperty("AsString"), [typeof(Guid)] = typeof(BsonValue).GetProperty("AsGuid") }; public static readonly PropertyInfo DocumentItemProperty = (from x in typeof(BsonDocument).GetProperties() where x.Name == "Item" && x.GetGetMethod().GetParameters().First() .ParameterType == typeof(string) select x).First(); private static readonly Dictionary _cacheName = new Dictionary(); public static object CreateInstance(Type type) { try { if (_cacheCtor.TryGetValue(type, out var value)) { return value(null); } } catch (Exception inner) { throw LiteException.InvalidCtor(type, inner); } lock (_cacheCtor) { try { if (_cacheCtor.TryGetValue(type, out var value2)) { return value2(null); } TypeInfo typeInfo = type.GetTypeInfo(); if (typeInfo.IsClass) { _cacheCtor.Add(type, value2 = CreateClass(type)); } else { if (typeInfo.IsInterface) { if (typeInfo.IsGenericType) { Type genericTypeDefinition = type.GetGenericTypeDefinition(); if (genericTypeDefinition == typeof(ISet<>)) { return CreateInstance(GetGenericSetOfType(UnderlyingTypeOf(type))); } if (genericTypeDefinition == typeof(IDictionary<, >)) { Type k = type.GetGenericArguments()[0]; Type v = type.GetGenericArguments()[1]; return CreateInstance(GetGenericDictionaryOfType(k, v)); } if (genericTypeDefinition == typeof(IList<>) || genericTypeDefinition == typeof(ICollection<>) || genericTypeDefinition == typeof(IEnumerable<>) || typeof(IEnumerable).IsAssignableFrom(genericTypeDefinition)) { return CreateInstance(GetGenericListOfType(UnderlyingTypeOf(type))); } } throw LiteException.InvalidCtor(type, null); } _cacheCtor.Add(type, value2 = CreateStruct(type)); } return value2(null); } catch (Exception inner2) { throw LiteException.InvalidCtor(type, inner2); } } } public static bool IsNullable(Type type) { if (!type.GetTypeInfo().IsGenericType) { return false; } return type.GetGenericTypeDefinition().Equals(typeof(Nullable<>)); } public static Type UnderlyingTypeOf(Type type) { if (!type.GetTypeInfo().IsGenericType) { return type; } return type.GetGenericArguments()[0]; } public static Type GetGenericListOfType(Type type) { return typeof(List<>).MakeGenericType(type); } public static Type GetGenericSetOfType(Type type) { return typeof(HashSet<>).MakeGenericType(type); } public static Type GetGenericDictionaryOfType(Type k, Type v) { return typeof(Dictionary<, >).MakeGenericType(k, v); } public static Type GetListItemType(Type listType) { if (listType.IsArray) { return listType.GetElementType(); } Type[] interfaces = listType.GetInterfaces(); foreach (Type type in interfaces) { if (type.GetTypeInfo().IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>)) { return type.GetGenericArguments()[0]; } if (listType.GetTypeInfo().IsGenericType && type == typeof(IEnumerable)) { return listType.GetGenericArguments()[0]; } } return typeof(object); } public static bool IsEnumerable(Type type) { if (type == typeof(IEnumerable) || type.IsArray) { return true; } if (type == typeof(string)) { return false; } Type[] interfaces = type.GetInterfaces(); foreach (Type type2 in interfaces) { if (type2.GetTypeInfo().IsGenericType && type2.GetGenericTypeDefinition() == typeof(IEnumerable<>)) { return true; } } return false; } public static bool IsSimpleType(Type type) { if (!(type == typeof(string)) && !(type == typeof(bool)) && !(type == typeof(byte)) && !(type == typeof(sbyte)) && !(type == typeof(short)) && !(type == typeof(int)) && !(type == typeof(long)) && !(type == typeof(ushort)) && !(type == typeof(uint)) && !(type == typeof(ulong)) && !(type == typeof(double)) && !(type == typeof(float)) && !(type == typeof(decimal)) && !(type == typeof(ObjectId)) && !(type == typeof(DateTime))) { return type == typeof(Guid); } return true; } public static bool IsCollection(Type type) { if (!type.GetTypeInfo().IsGenericType || !type.GetGenericTypeDefinition().Equals(typeof(ICollection<>))) { return type.GetInterfaces().Any((Type x) => x == typeof(ICollection) || (x.GetTypeInfo().IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))); } return true; } public static bool IsDictionary(Type type) { if (!type.GetTypeInfo().IsGenericType || !type.GetGenericTypeDefinition().Equals(typeof(IDictionary<, >))) { return type.GetInterfaces().Any((Type x) => x == typeof(IDictionary) || (x.GetTypeInfo().IsGenericType && x.GetGenericTypeDefinition().Equals(typeof(IDictionary<, >)))); } return true; } public static MemberInfo SelectMember(IEnumerable members, params Func[] predicates) { foreach (Func predicate in predicates) { MemberInfo memberInfo = members.FirstOrDefault(predicate); if (memberInfo != null) { return memberInfo; } } return null; } public static string MethodName(MethodInfo method, int skipParameters = 0) { lock (_cacheName) { if (_cacheName.TryGetValue(method, out var value)) { return value; } value = MethodNameInternal(method, skipParameters); _cacheName.Add(method, value); return value; } } private static string MethodNameInternal(MethodInfo method, int skipParameters = 0) { StringBuilder stringBuilder = new StringBuilder(method.Name + "("); int num = 0; foreach (ParameterInfo item in method.GetParameters().Skip(skipParameters)) { if (num++ > 0) { stringBuilder.Append(","); } stringBuilder.Append(FriendlyTypeName(item.ParameterType)); if (item.ParameterType.GetTypeInfo().IsGenericType) { Type[] genericArguments = item.ParameterType.GetGenericTypeDefinition().GetGenericArguments(); stringBuilder.Append("<"); stringBuilder.Append(string.Join(",", genericArguments.Select((Type x) => FriendlyTypeName(x)))); stringBuilder.Append(">"); } } stringBuilder.Append(")"); return stringBuilder.ToString(); } private static string FriendlyTypeName(Type type) { int num = type.Name.IndexOf("`"); switch (type.FullName) { case "System.Object": return "object"; case "System.String": return "string"; case "System.Boolean": return "bool"; case "System.Byte": return "byte"; case "System.Char": return "char"; case "System.Decimal": return "decimal"; case "System.Double": return "double"; case "System.Int16": return "short"; case "System.Int32": return "int"; case "System.Int64": return "long"; case "System.SByte": return "sbyte"; case "System.Single": return "float"; case "System.UInt16": return "ushort"; case "System.UInt32": return "uint"; case "System.UInt64": return "ulong"; default: if (num <= 0) { return type.Name; } return type.Name.Substring(0, num); } } public static CreateObject CreateClass(Type type) { ParameterExpression parameterExpression = Expression.Parameter(typeof(BsonDocument), "_doc"); return Expression.Lambda(Expression.New(type), new ParameterExpression[1] { parameterExpression }).Compile(); } public static CreateObject CreateStruct(Type type) { ParameterExpression parameterExpression = Expression.Parameter(typeof(BsonDocument), "_doc"); return Expression.Lambda(Expression.Convert(Expression.New(type), typeof(object)), new ParameterExpression[1] { parameterExpression }).Compile(); } public static GenericGetter CreateGenericGetter(Type type, MemberInfo memberInfo) { if (memberInfo == null) { throw new ArgumentNullException("memberInfo"); } if (memberInfo is PropertyInfo && !(memberInfo as PropertyInfo).CanRead) { return null; } ParameterExpression parameterExpression = Expression.Parameter(typeof(object), "o"); return Expression.Lambda(Expression.Convert(Expression.MakeMemberAccess(Expression.Convert(parameterExpression, memberInfo.DeclaringType), memberInfo), typeof(object)), new ParameterExpression[1] { parameterExpression }).Compile(); } public static GenericSetter CreateGenericSetter(Type type, MemberInfo memberInfo) { if (memberInfo == null) { throw new ArgumentNullException("memberInfo"); } FieldInfo fieldInfo = memberInfo as FieldInfo; PropertyInfo propertyInfo = memberInfo as PropertyInfo; if (memberInfo is PropertyInfo && !propertyInfo.CanWrite) { return null; } if (type.GetTypeInfo().IsValueType) { if (!(memberInfo is FieldInfo)) { return delegate(object t, object v) { propertyInfo.SetValue(t, v, null); }; } return fieldInfo.SetValue; } Type type2 = ((memberInfo is PropertyInfo) ? propertyInfo.PropertyType : fieldInfo.FieldType); ParameterExpression parameterExpression = Expression.Parameter(typeof(object), "obj"); ParameterExpression parameterExpression2 = Expression.Parameter(typeof(object), "val"); UnaryExpression expression = Expression.Convert(parameterExpression, type); UnaryExpression right = Expression.ConvertChecked(parameterExpression2, type2); return Expression.Lambda(Expression.Convert(Expression.Assign((memberInfo is PropertyInfo) ? Expression.Property(expression, propertyInfo) : Expression.Field(expression, fieldInfo), right), typeof(object)), new ParameterExpression[2] { parameterExpression, parameterExpression2 }).Compile(); } } public class DefaultTypeNameBinder : ITypeNameBinder { private static readonly HashSet _disallowedTypeNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "System.Workflow.ComponentModel.AppSettings", "System.Core", "WinRT.BaseActivationFactory", "System.Data", "System.Windows.Data.ObjectDataProvider", "System.CodeDom.Compiler.CompilerResults", "System.Collections.ArrayList", "System.Diagnostics.Process", "System.Diagnostics.ProcessStartInfo", "System.Management.Automation", "System.Windows.Markup.XamlReader", "System.Web.Security.RolePrincipal", "System.Security.Principal.WindowsIdentity", "System.Security.Principal.WindowsPrincipal", "Microsoft.VisualStudio.Text.Formatting.TextFormattingRunProperties", "System.Drawing.Design.ToolboxItemContainer", "Microsoft.IdentityModel.Claims.WindowsClaimsIdentity", "System.Resources.ResXResourceReader", "System.Resources.ResXResourceWriter", "System.Windows.Forms", "Microsoft.ApplicationId.Framework.InfiniteProgressPage", "Microsoft.VisualBasic.Logging.FileLogTraceListener", "Grpc.Core.Internal.UnmanagedLibrary", "MongoDB.Libmongocrypt.LibraryLoader+WindowsLibrary", "Xunit.Xunit1Executor", "Apache.NMS.ActiveMQ.Commands.ActiveMQObjectMessage", "Apache.NMS.ActiveMQ.Transport.Failover.FailoverTransport", "Apache.NMS.ActiveMQ.Util.IdGenerator", "Xunit.Sdk.TestFrameworkDiscoverer+PreserveWorkingFolder", "Xunit.Xunit1AssemblyInfo", "Amazon.Runtime.Internal.Util.OptimisticLockedTextFile", "Microsoft.Azure.Cosmos.Query.Core.QueryPlan.QueryPartitionProvider", "NLog.Internal.FileAppenders.SingleProcessFileAppender", "NLog.Targets.FileTarget", "Google.Apis.Util.Store.FileDataStore" }; public static DefaultTypeNameBinder Instance { get; } = new DefaultTypeNameBinder(); private DefaultTypeNameBinder() { } public string GetName(Type type) { return type.FullName + ", " + type.GetTypeInfo().Assembly.GetName().Name; } public Type GetType(string name) { Type type = Type.GetType(name); if (type == null) { return null; } if (_disallowedTypeNames.Contains(type.FullName)) { throw LiteException.IllegalDeserializationType(type.FullName); } return type; } } public interface ITypeNameBinder { string GetName(Type type); Type GetType(string name); } public class SharedDataReader : IBsonDataReader, IDisposable { private readonly IBsonDataReader _reader; private readonly Action _dispose; private bool _disposed; public BsonValue this[string field] => _reader[field]; public string Collection => _reader.Collection; public BsonValue Current => _reader.Current; public bool HasValues => _reader.HasValues; public SharedDataReader(IBsonDataReader reader, Action dispose) { _reader = reader; _dispose = dispose; } public bool Read() { return _reader.Read(); } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } ~SharedDataReader() { Dispose(disposing: false); } protected virtual void Dispose(bool disposing) { if (!_disposed) { _disposed = true; if (disposing) { _reader.Dispose(); _dispose(); } } } } public class SharedEngine : ILiteEngine, IDisposable { private readonly EngineSettings _settings; private readonly Mutex _mutex; private LiteEngine _engine; private bool _transactionRunning; public SharedEngine(EngineSettings settings) { _settings = settings; string text = Path.GetFullPath(settings.Filename).ToLower().Sha1(); try { _mutex = new Mutex(initiallyOwned: false, "Global\\" + text + ".Mutex"); } catch (NotSupportedException inner) { throw new PlatformNotSupportedException("Shared mode is not supported in platforms that do not implement named mutex.", inner); } } private void OpenDatabase() { try { _mutex.WaitOne(); } catch (AbandonedMutexException) { } if (!_transactionRunning && _engine == null) { try { _engine = new LiteEngine(_settings); } catch { _mutex.ReleaseMutex(); throw; } } } private void CloseDatabase() { if (!_transactionRunning && _engine != null) { _engine.Dispose(); _engine = null; } _mutex.ReleaseMutex(); } public bool BeginTrans() { OpenDatabase(); try { _transactionRunning = _engine.BeginTrans(); return _transactionRunning; } catch { CloseDatabase(); throw; } } public bool Commit() { if (_engine == null) { return false; } try { return _engine.Commit(); } finally { _transactionRunning = false; CloseDatabase(); } } public bool Rollback() { if (_engine == null) { return false; } try { return _engine.Rollback(); } finally { _transactionRunning = false; CloseDatabase(); } } public IBsonDataReader Query(string collection, Query query) { OpenDatabase(); return new SharedDataReader(_engine.Query(collection, query), delegate { CloseDatabase(); }); } public BsonValue Pragma(string name) { OpenDatabase(); try { return _engine.Pragma(name); } finally { CloseDatabase(); } } public bool Pragma(string name, BsonValue value) { OpenDatabase(); try { return _engine.Pragma(name, value); } finally { CloseDatabase(); } } public int Checkpoint() { OpenDatabase(); try { return _engine.Checkpoint(); } finally { CloseDatabase(); } } public long Rebuild(RebuildOptions options) { OpenDatabase(); try { return _engine.Rebuild(options); } finally { CloseDatabase(); } } public int Insert(string collection, IEnumerable docs, BsonAutoId autoId) { OpenDatabase(); try { return _engine.Insert(collection, docs, autoId); } finally { CloseDatabase(); } } public int Update(string collection, IEnumerable docs) { OpenDatabase(); try { return _engine.Update(collection, docs); } finally { CloseDatabase(); } } public int UpdateMany(string collection, BsonExpression extend, BsonExpression predicate) { OpenDatabase(); try { return _engine.UpdateMany(collection, extend, predicate); } finally { CloseDatabase(); } } public int Upsert(string collection, IEnumerable docs, BsonAutoId autoId) { OpenDatabase(); try { return _engine.Upsert(collection, docs, autoId); } finally { CloseDatabase(); } } public int Delete(string collection, IEnumerable ids) { OpenDatabase(); try { return _engine.Delete(collection, ids); } finally { CloseDatabase(); } } public int DeleteMany(string collection, BsonExpression predicate) { OpenDatabase(); try { return _engine.DeleteMany(collection, predicate); } finally { CloseDatabase(); } } public bool DropCollection(string name) { OpenDatabase(); try { return _engine.DropCollection(name); } finally { CloseDatabase(); } } public bool RenameCollection(string name, string newName) { OpenDatabase(); try { return _engine.RenameCollection(name, newName); } finally { CloseDatabase(); } } public bool DropIndex(string collection, string name) { OpenDatabase(); try { return _engine.DropIndex(collection, name); } finally { CloseDatabase(); } } public bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique) { OpenDatabase(); try { return _engine.EnsureIndex(collection, name, expression, unique); } finally { CloseDatabase(); } } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } ~SharedEngine() { Dispose(disposing: false); } protected virtual void Dispose(bool disposing) { if (disposing && _engine != null) { _engine.Dispose(); _engine = null; _mutex.ReleaseMutex(); } } } internal class SqlParser { private readonly ILiteEngine _engine; private readonly Tokenizer _tokenizer; private readonly BsonDocument _parameters; private readonly Lazy _collation; private BsonDataReader ParseBegin() { _tokenizer.ReadToken().Expect("BEGIN"); Token token = _tokenizer.ReadToken().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon); if (token.Is("TRANS") || token.Is("TRANSACTION")) { _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); } return new BsonDataReader(_engine.BeginTrans()); } private BsonDataReader ParseCheckpoint() { _tokenizer.ReadToken().Expect("CHECKPOINT"); _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.Checkpoint()); } private BsonDataReader ParseCommit() { _tokenizer.ReadToken().Expect("COMMIT"); Token token = _tokenizer.ReadToken().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon); if (token.Is("TRANS") || token.Is("TRANSACTION")) { _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); } return new BsonDataReader(_engine.Commit()); } private BsonDataReader ParseCreate() { _tokenizer.ReadToken().Expect("CREATE"); Token token = _tokenizer.ReadToken().Expect(TokenType.Word); bool flag = token.Is("UNIQUE"); if (flag) { token = _tokenizer.ReadToken(); } token.Expect("INDEX"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect("ON"); string value2 = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect(TokenType.OpenParenthesis); BsonExpression expression = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, new BsonDocument()); _tokenizer.ReadToken().Expect(TokenType.CloseParenthesis); _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.EnsureIndex(value2, value, expression, flag)); } private BsonDataReader ParseDelete() { _tokenizer.ReadToken().Expect("DELETE"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; BsonExpression predicate = null; if (_tokenizer.LookAhead().Is("WHERE")) { _tokenizer.ReadToken(); predicate = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); } _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); _tokenizer.ReadToken(); return new BsonDataReader(_engine.DeleteMany(value, predicate)); } private BsonDataReader ParseDrop() { _tokenizer.ReadToken().Expect("DROP"); Token token = _tokenizer.ReadToken().Expect(TokenType.Word); if (token.Is("INDEX")) { string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect(TokenType.Period); string value2 = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.DropIndex(value, value2)); } if (token.Is("COLLECTION")) { string value3 = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.DropCollection(value3)); } throw LiteException.UnexpectedToken(token, "INDEX|COLLECTION"); } private BsonDataReader ParseInsert() { _tokenizer.ReadToken().Expect("INSERT"); _tokenizer.ReadToken().Expect("INTO"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; BsonAutoId autoId = ParseWithAutoId(); _tokenizer.ReadToken().Expect("VALUES"); IEnumerable docs = ParseListOfDocuments(); return new BsonDataReader(_engine.Insert(value, docs, autoId)); } private BsonAutoId ParseWithAutoId() { if (_tokenizer.LookAhead().Type == TokenType.Colon) { _tokenizer.ReadToken(); Token token = _tokenizer.ReadToken().Expect(TokenType.Word); return token.Value.ToUpper() switch { "GUID" => BsonAutoId.Guid, "INT" => BsonAutoId.Int32, "LONG" => BsonAutoId.Int64, "OBJECTID" => BsonAutoId.ObjectId, _ => throw LiteException.UnexpectedToken(token, "DATE, GUID, INT, LONG, OBJECTID"), }; } return BsonAutoId.ObjectId; } private IEnumerable ParseListOfExpressions() { while (true) { yield return BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); if (_tokenizer.LookAhead().Type == TokenType.Comma) { _tokenizer.ReadToken(); continue; } break; } } private IEnumerable ParseListOfDocuments() { JsonReader reader = new JsonReader(_tokenizer); Token token; while (true) { BsonValue bsonValue = reader.Deserialize(); if (bsonValue.IsDocument) { yield return bsonValue as BsonDocument; token = _tokenizer.LookAhead(); if (token.Type != TokenType.Comma) { break; } _tokenizer.ReadToken(); continue; } throw LiteException.UnexpectedToken("Value must be a valid document", _tokenizer.Current); } token.Expect(TokenType.EOF, TokenType.SemiColon); _tokenizer.ReadToken(); } private IBsonDataReader ParsePragma() { _tokenizer.ReadToken().Expect("PRAGMA"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; Token token = _tokenizer.LookAhead(); if (token.Type == TokenType.EOF || token.Type == TokenType.SemiColon) { _tokenizer.ReadToken(); return new BsonDataReader(_engine.Pragma(value)); } if (token.Type == TokenType.Equals) { _tokenizer.ReadToken().Expect(TokenType.Equals); BsonValue value2 = new JsonReader(_tokenizer).Deserialize(); _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.Pragma(value, value2)); } throw LiteException.UnexpectedToken(token); } private BsonDataReader ParseRebuild() { _tokenizer.ReadToken().Expect("REBUILD"); RebuildOptions rebuildOptions = new RebuildOptions(); Token token = _tokenizer.LookAhead(); if (token.Type == TokenType.EOF || token.Type == TokenType.SemiColon) { rebuildOptions = null; _tokenizer.ReadToken(); } else { BsonValue bsonValue = new JsonReader(_tokenizer).Deserialize(); if (!bsonValue.IsDocument) { throw LiteException.UnexpectedToken(token); } if (bsonValue["password"].IsString) { rebuildOptions.Password = bsonValue["password"]; } if (bsonValue["collation"].IsString) { rebuildOptions.Collation = new Collation(bsonValue["collation"].AsString); } } return new BsonDataReader((int)_engine.Rebuild(rebuildOptions)); } private BsonDataReader ParseRename() { _tokenizer.ReadToken().Expect("RENAME"); _tokenizer.ReadToken().Expect("COLLECTION"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect("TO"); string value2 = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.RenameCollection(value, value2)); } private BsonDataReader ParseRollback() { _tokenizer.ReadToken().Expect("ROLLBACK"); Token token = _tokenizer.ReadToken().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon); if (token.Is("TRANS") || token.Is("TRANSACTION")) { _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); } return new BsonDataReader(_engine.Rollback()); } private IBsonDataReader ParseSelect() { Query query = new Query(); Token token = _tokenizer.ReadToken(); query.ExplainPlan = token.Is("EXPLAIN"); if (query.ExplainPlan) { token = _tokenizer.ReadToken(); } token.Expect("SELECT"); query.Select = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.SelectDocument, _parameters); Token token2 = _tokenizer.ReadToken(); if (token2.Type == TokenType.EOF || token2.Type == TokenType.SemiColon) { IEnumerable source = query.Select.Execute(_collation.Value); string defaultName = "expr"; return new BsonDataReader(source.Select((BsonValue x) => (!x.IsDocument) ? new BsonDocument { [defaultName] = x } : x.AsDocument).FirstOrDefault()); } if (token2.Is("INTO")) { query.Into = ParseCollection(_tokenizer); query.IntoAutoId = ParseWithAutoId(); _tokenizer.ReadToken().Expect("FROM"); } else { token2.Expect("FROM"); } string collection = ParseCollection(_tokenizer); if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("INCLUDE")) { _tokenizer.ReadToken(); foreach (BsonExpression item2 in ParseListOfExpressions()) { query.Includes.Add(item2); } } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("WHERE")) { _tokenizer.ReadToken(); BsonExpression item = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); query.Where.Add(item); } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("GROUP")) { _tokenizer.ReadToken(); _tokenizer.ReadToken().Expect("BY"); BsonExpression groupBy = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); query.GroupBy = groupBy; if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("HAVING")) { _tokenizer.ReadToken(); BsonExpression having = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); query.Having = having; } } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("ORDER")) { _tokenizer.ReadToken(); _tokenizer.ReadToken().Expect("BY"); BsonExpression orderBy = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); int order = 1; Token token3 = _tokenizer.LookAhead(); if (token3.Is("ASC") || token3.Is("DESC")) { order = (_tokenizer.ReadToken().Is("ASC") ? 1 : (-1)); } query.OrderBy = orderBy; query.Order = order; } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("LIMIT")) { _tokenizer.ReadToken(); string value = _tokenizer.ReadToken().Expect(TokenType.Int).Value; query.Limit = Convert.ToInt32(value); } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("OFFSET")) { _tokenizer.ReadToken(); string value2 = _tokenizer.ReadToken().Expect(TokenType.Int).Value; query.Offset = Convert.ToInt32(value2); } if (_tokenizer.LookAhead().Expect(TokenType.Word, TokenType.EOF, TokenType.SemiColon).Is("FOR")) { _tokenizer.ReadToken(); _tokenizer.ReadToken().Expect("UPDATE"); query.ForUpdate = true; } _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return _engine.Query(collection, query); } public static string ParseCollection(Tokenizer tokenizer) { string name; BsonValue options; return ParseCollection(tokenizer, out name, out options); } public static string ParseCollection(Tokenizer tokenizer, out string name, out BsonValue options) { name = tokenizer.ReadToken().Expect(TokenType.Word).Value; if (name.StartsWith("$")) { if (tokenizer.LookAhead().Type == TokenType.OpenParenthesis) { tokenizer.ReadToken(); if (tokenizer.LookAhead().Type == TokenType.CloseParenthesis) { options = null; } else { options = new JsonReader(tokenizer).Deserialize(); } tokenizer.ReadToken().Expect(TokenType.CloseParenthesis); } else { options = null; } } else { options = null; } return name + ((options == null) ? "" : ("(" + JsonSerializer.Serialize(options) + ")")); } private BsonDataReader ParseUpdate() { _tokenizer.ReadToken().Expect("UPDATE"); string value = _tokenizer.ReadToken().Expect(TokenType.Word).Value; _tokenizer.ReadToken().Expect("SET"); BsonExpression transform = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.UpdateDocument, _parameters); BsonExpression predicate = null; if (_tokenizer.LookAhead().Is("WHERE")) { _tokenizer.ReadToken(); predicate = BsonExpression.Create(_tokenizer, BsonExpressionParserMode.Full, _parameters); } _tokenizer.ReadToken().Expect(TokenType.EOF, TokenType.SemiColon); return new BsonDataReader(_engine.UpdateMany(value, transform, predicate)); } public SqlParser(ILiteEngine engine, Tokenizer tokenizer, BsonDocument parameters) { _engine = engine; _tokenizer = tokenizer; _parameters = parameters ?? new BsonDocument(); _collation = new Lazy(() => new Collation(_engine.Pragma("COLLATION"))); } public IBsonDataReader Execute() { Token token = _tokenizer.LookAhead().Expect(TokenType.Word); switch (token.Value.ToUpper()) { case "SELECT": case "EXPLAIN": return ParseSelect(); case "INSERT": return ParseInsert(); case "DELETE": return ParseDelete(); case "UPDATE": return ParseUpdate(); case "DROP": return ParseDrop(); case "RENAME": return ParseRename(); case "CREATE": return ParseCreate(); case "CHECKPOINT": return ParseCheckpoint(); case "REBUILD": return ParseRebuild(); case "BEGIN": return ParseBegin(); case "ROLLBACK": return ParseRollback(); case "COMMIT": return ParseCommit(); case "PRAGMA": return ParsePragma(); default: throw LiteException.UnexpectedToken(token); } } } public interface ILiteStorage { LiteFileInfo FindById(TFileId id); IEnumerable> Find(BsonExpression predicate); IEnumerable> Find(string predicate, BsonDocument parameters); IEnumerable> Find(string predicate, params BsonValue[] args); IEnumerable> Find(Expression, bool>> predicate); IEnumerable> FindAll(); bool Exists(TFileId id); LiteFileStream OpenWrite(TFileId id, string filename, BsonDocument metadata = null); LiteFileInfo Upload(TFileId id, string filename, Stream stream, BsonDocument metadata = null); LiteFileInfo Upload(TFileId id, string filename); bool SetMetadata(TFileId id, BsonDocument metadata); LiteFileStream OpenRead(TFileId id); LiteFileInfo Download(TFileId id, Stream stream); LiteFileInfo Download(TFileId id, string filename, bool overwritten); bool Delete(TFileId id); } public class LiteFileInfo { private BsonValue _fileId; private ILiteCollection> _files; private ILiteCollection _chunks; public TFileId Id { get; internal set; } [BsonField("filename")] public string Filename { get; internal set; } [BsonField("mimeType")] public string MimeType { get; internal set; } [BsonField("length")] public long Length { get; internal set; } [BsonField("chunks")] public int Chunks { get; internal set; } [BsonField("uploadDate")] public DateTime UploadDate { get; internal set; } = DateTime.Now; [BsonField("metadata")] public BsonDocument Metadata { get; set; } = new BsonDocument(); internal void SetReference(BsonValue fileId, ILiteCollection> files, ILiteCollection chunks) { _fileId = fileId; _files = files; _chunks = chunks; } public LiteFileStream OpenRead() { return new LiteFileStream(_files, _chunks, this, _fileId, FileAccess.Read); } public LiteFileStream OpenWrite() { return new LiteFileStream(_files, _chunks, this, _fileId, FileAccess.Write); } public void CopyTo(Stream stream) { if (stream == null) { throw new ArgumentNullException("stream"); } using LiteFileStream liteFileStream = OpenRead(); liteFileStream.CopyTo(stream); } public void SaveAs(string filename, bool overwritten = true) { if (filename.IsNullOrWhiteSpace()) { throw new ArgumentNullException("filename"); } using FileStream destination = File.Open(filename, (!overwritten) ? FileMode.CreateNew : FileMode.Create); using LiteFileStream liteFileStream = OpenRead(); liteFileStream.CopyTo(destination); } } public class LiteFileStream : Stream { public const int MAX_CHUNK_SIZE = 261120; private readonly ILiteCollection> _files; private readonly ILiteCollection _chunks; private readonly LiteFileInfo _file; private readonly BsonValue _fileId; private readonly FileAccess _mode; private long _streamPosition; private int _currentChunkIndex; private byte[] _currentChunkData; private int _positionInChunk; private MemoryStream _buffer; private bool _disposed; private Dictionary _chunkLengths = new Dictionary(); public LiteFileInfo FileInfo => _file; public override long Length => _file.Length; public override bool CanRead => _mode == FileAccess.Read; public override bool CanWrite => _mode == FileAccess.Write; public override bool CanSeek => _mode == FileAccess.Read; public override long Position { get { return _streamPosition; } set { if (_mode == FileAccess.Read) { SetReadStreamPosition(value); return; } throw new NotSupportedException(); } } internal LiteFileStream(ILiteCollection> files, ILiteCollection chunks, LiteFileInfo file, BsonValue fileId, FileAccess mode) { _files = files; _chunks = chunks; _file = file; _fileId = fileId; _mode = mode; switch (mode) { case FileAccess.Read: _currentChunkData = GetChunkData(_currentChunkIndex); break; case FileAccess.Write: _buffer = new MemoryStream(261120); if (_file.Length > 0) { Constants.ENSURE(_chunks.DeleteMany("_id BETWEEN { f: @0, n: 0 } AND { f: @0, n: 99999999 }", _fileId) == _file.Chunks); _file.Length = 0L; _file.Chunks = 0; } break; } } public override long Seek(long offset, SeekOrigin origin) { if (_mode == FileAccess.Write) { throw new NotSupportedException(); } switch (origin) { case SeekOrigin.Begin: SetReadStreamPosition(offset); break; case SeekOrigin.Current: SetReadStreamPosition(_streamPosition + offset); break; case SeekOrigin.End: SetReadStreamPosition(Length + offset); break; } return _streamPosition; } protected override void Dispose(bool disposing) { base.Dispose(disposing); if (!_disposed) { if (disposing && CanWrite) { Flush(); _buffer?.Dispose(); } _disposed = true; } } public override void SetLength(long value) { throw new NotSupportedException(); } public override int Read(byte[] buffer, int offset, int count) { if (_mode != FileAccess.Read) { throw new NotSupportedException(); } if (_streamPosition == Length) { return 0; } int num = count; while (_currentChunkData != null && num > 0) { int num2 = Math.Min(num, _currentChunkData.Length - _positionInChunk); Buffer.BlockCopy(_currentChunkData, _positionInChunk, buffer, offset, num2); _positionInChunk += num2; num -= num2; offset += num2; _streamPosition += num2; if (_positionInChunk >= _currentChunkData.Length) { _positionInChunk = 0; _currentChunkData = GetChunkData(++_currentChunkIndex); } } return count - num; } private byte[] GetChunkData(int index) { byte[] array = _chunks.FindOne("_id = { f: @0, n: @1 }", _fileId, index)?["data"].AsBinary; if (array != null) { _chunkLengths[index] = array.Length; } return array; } private void SetReadStreamPosition(long newPosition) { if (newPosition < 0) { throw new ArgumentOutOfRangeException(); } if (newPosition >= Length) { _streamPosition = Length; return; } _streamPosition = newPosition; long num = 0L; int num2 = _currentChunkIndex; int num3 = 0; while (num <= _streamPosition) { if (_chunkLengths.TryGetValue(num3, out var value)) { num += value; } else { num2 = num3; _currentChunkData = GetChunkData(num3); num += _currentChunkData.Length; } num3++; } num3--; num -= _chunkLengths[num3]; _positionInChunk = (int)(_streamPosition - num); _currentChunkIndex = num3; if (num2 != _currentChunkIndex) { _currentChunkData = GetChunkData(_currentChunkIndex); } } public override void Write(byte[] buffer, int offset, int count) { _streamPosition += count; _buffer.Write(buffer, offset, count); if (_buffer.Length >= 261120) { WriteChunks(flush: false); } } public override void Flush() { WriteChunks(flush: true); } private void WriteChunks(bool flush) { byte[] array = new byte[261120]; int num = 0; _buffer.Seek(0L, SeekOrigin.Begin); while ((num = _buffer.Read(array, 0, 261120)) > 0) { BsonDocument bsonDocument = new BsonDocument { ["_id"] = new BsonDocument { ["f"] = _fileId, ["n"] = _file.Chunks++ } }; if (num != 261120) { byte[] array2 = new byte[num]; Buffer.BlockCopy(array, 0, array2, 0, num); bsonDocument["data"] = array2; } else { bsonDocument["data"] = array; } _chunks.Insert(bsonDocument); } if (flush) { _file.UploadDate = DateTime.Now; _file.Length = _streamPosition; _files.Upsert(_file); } _buffer = new MemoryStream(); } } public class LiteStorage : ILiteStorage { private readonly ILiteDatabase _db; private readonly ILiteCollection> _files; private readonly ILiteCollection _chunks; public LiteStorage(ILiteDatabase db, string filesCollection, string chunksCollection) { _db = db; _files = db.GetCollection>(filesCollection); _chunks = db.GetCollection(chunksCollection); } public LiteFileInfo FindById(TFileId id) { if (id == null) { throw new ArgumentNullException("id"); } BsonValue bsonValue = _db.Mapper.Serialize(typeof(TFileId), id); LiteFileInfo liteFileInfo = _files.FindById(bsonValue); if (liteFileInfo == null) { return null; } liteFileInfo.SetReference(bsonValue, _files, _chunks); return liteFileInfo; } public IEnumerable> Find(BsonExpression predicate) { ILiteQueryable> liteQueryable = _files.Query(); if (predicate != null) { liteQueryable = liteQueryable.Where(predicate); } foreach (LiteFileInfo item in liteQueryable.ToEnumerable()) { BsonValue fileId = _db.Mapper.Serialize(typeof(TFileId), item.Id); item.SetReference(fileId, _files, _chunks); yield return item; } } public IEnumerable> Find(string predicate, BsonDocument parameters) { return Find(BsonExpression.Create(predicate, parameters)); } public IEnumerable> Find(string predicate, params BsonValue[] args) { return Find(BsonExpression.Create(predicate, args)); } public IEnumerable> Find(Expression, bool>> predicate) { return Find(_db.Mapper.GetExpression(predicate)); } public IEnumerable> FindAll() { return Find((BsonExpression)null); } public bool Exists(TFileId id) { if (id == null) { throw new ArgumentNullException("id"); } BsonValue bsonValue = _db.Mapper.Serialize(typeof(TFileId), id); return _files.Exists("_id = @0", bsonValue); } public LiteFileStream OpenWrite(TFileId id, string filename, BsonDocument metadata = null) { BsonValue fileId = _db.Mapper.Serialize(typeof(TFileId), id); LiteFileInfo liteFileInfo = FindById(id); if (liteFileInfo == null) { liteFileInfo = new LiteFileInfo { Id = id, Filename = Path.GetFileName(filename), MimeType = MimeTypeConverter.GetMimeType(filename), Metadata = (metadata ?? new BsonDocument()) }; liteFileInfo.SetReference(fileId, _files, _chunks); } else { liteFileInfo.Filename = Path.GetFileName(filename); liteFileInfo.MimeType = MimeTypeConverter.GetMimeType(filename); liteFileInfo.Metadata = metadata ?? liteFileInfo.Metadata; } return liteFileInfo.OpenWrite(); } public LiteFileInfo Upload(TFileId id, string filename, Stream stream, BsonDocument metadata = null) { using LiteFileStream liteFileStream = OpenWrite(id, filename, metadata); stream.CopyTo(liteFileStream); return liteFileStream.FileInfo; } public LiteFileInfo Upload(TFileId id, string filename) { if (filename.IsNullOrWhiteSpace()) { throw new ArgumentNullException("filename"); } using FileStream stream = File.OpenRead(filename); return Upload(id, Path.GetFileName(filename), stream); } public bool SetMetadata(TFileId id, BsonDocument metadata) { LiteFileInfo liteFileInfo = FindById(id); if (liteFileInfo == null) { return false; } liteFileInfo.Metadata = metadata ?? new BsonDocument(); _files.Update(liteFileInfo); return true; } public LiteFileStream OpenRead(TFileId id) { return (FindById(id) ?? throw LiteException.FileNotFound(id.ToString())).OpenRead(); } public LiteFileInfo Download(TFileId id, Stream stream) { LiteFileInfo obj = FindById(id) ?? throw LiteException.FileNotFound(id.ToString()); obj.CopyTo(stream); return obj; } public LiteFileInfo Download(TFileId id, string filename, bool overwritten) { LiteFileInfo obj = FindById(id) ?? throw LiteException.FileNotFound(id.ToString()); obj.SaveAs(filename, overwritten); return obj; } public bool Delete(TFileId id) { if (id == null) { throw new ArgumentNullException("id"); } BsonValue bsonValue = _db.Mapper.Serialize(typeof(TFileId), id); bool flag = _files.Delete(bsonValue); if (flag) { _chunks.DeleteMany("_id BETWEEN { f: @0, n: 0} AND {f: @0, n: @1 }", bsonValue, int.MaxValue); } return flag; } } public class ConnectionString { private readonly Dictionary _values; public ConnectionType Connection { get; set; } public string Filename { get; set; } = ""; public string Password { get; set; } public long InitialSize { get; set; } public bool ReadOnly { get; set; } public bool Upgrade { get; set; } public bool AutoRebuild { get; set; } public Collation Collation { get; set; } public string this[string key] => _values.GetOrDefault(key); public ConnectionString() { _values = new Dictionary(StringComparer.OrdinalIgnoreCase); } public ConnectionString(string connectionString) : this() { if (string.IsNullOrEmpty(connectionString)) { throw new ArgumentNullException("connectionString"); } if (connectionString.Contains("=")) { _values.ParseKeyValue(connectionString); } else { _values["filename"] = connectionString; } Connection = _values.GetValue("connection", Connection); Filename = _values.GetValue("filename", Filename).Trim(); Password = _values.GetValue("password", Password); if (Password == string.Empty) { Password = null; } InitialSize = _values.GetFileSize("initial size", InitialSize); ReadOnly = _values.GetValue("readonly", ReadOnly); Collation = (_values.ContainsKey("collation") ? new Collation(_values.GetValue("collation")) : Collation); Upgrade = _values.GetValue("upgrade", Upgrade); AutoRebuild = _values.GetValue("auto-rebuild", AutoRebuild); } internal ILiteEngine CreateEngine(Action engineSettingsAction = null) { EngineSettings engineSettings = new EngineSettings { Filename = Filename, Password = Password, InitialSize = InitialSize, ReadOnly = ReadOnly, Collation = Collation, Upgrade = Upgrade, AutoRebuild = AutoRebuild }; engineSettingsAction?.Invoke(engineSettings); if (Connection == ConnectionType.Direct) { return new LiteEngine(engineSettings); } if (Connection == ConnectionType.Shared) { return new SharedEngine(engineSettings); } throw new NotImplementedException(); } } public enum ConnectionType { Direct, Shared } public class Query { public const int Ascending = 1; public const int Descending = -1; public BsonExpression Select { get; set; } = BsonExpression.Root; public List Includes { get; } = new List(); public List Where { get; } = new List(); public BsonExpression OrderBy { get; set; } public int Order { get; set; } = 1; public BsonExpression GroupBy { get; set; } public BsonExpression Having { get; set; } public int Offset { get; set; } public int Limit { get; set; } = int.MaxValue; public bool ForUpdate { get; set; } public string Into { get; set; } public BsonAutoId IntoAutoId { get; set; } = BsonAutoId.ObjectId; public bool ExplainPlan { get; set; } public static Query All() { return new Query(); } public static Query All(int order = 1) { return new Query { OrderBy = "_id", Order = order }; } public static Query All(string field, int order = 1) { return new Query { OrderBy = field, Order = order }; } public static BsonExpression EQ(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} = {value ?? BsonValue.Null}"); } public static BsonExpression LT(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} < {value ?? BsonValue.Null}"); } public static BsonExpression LTE(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} <= {value ?? BsonValue.Null}"); } public static BsonExpression GT(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} > {value ?? BsonValue.Null}"); } public static BsonExpression GTE(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} >= {value ?? BsonValue.Null}"); } public static BsonExpression Between(string field, BsonValue start, BsonValue end) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} BETWEEN {start ?? BsonValue.Null} AND {end ?? BsonValue.Null}"); } public static BsonExpression StartsWith(string field, string value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } if (value.IsNullOrWhiteSpace()) { throw new ArgumentNullException("value"); } return BsonExpression.Create(string.Format("{0} LIKE {1}", field, new BsonValue(value + "%"))); } public static BsonExpression Contains(string field, string value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } if (value.IsNullOrEmpty()) { throw new ArgumentNullException("value"); } return BsonExpression.Create(string.Format("{0} LIKE {1}", field, new BsonValue("%" + value + "%"))); } public static BsonExpression Not(string field, BsonValue value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } return BsonExpression.Create($"{field} != {value ?? BsonValue.Null}"); } public static BsonExpression In(string field, BsonArray value) { if (field.IsNullOrWhiteSpace()) { throw new ArgumentNullException("field"); } if (value == null) { throw new ArgumentNullException("value"); } return BsonExpression.Create($"{field} IN {value}"); } public static BsonExpression In(string field, params BsonValue[] values) { return In(field, new BsonArray(values)); } public static BsonExpression In(string field, IEnumerable values) { return In(field, new BsonArray(values)); } public static QueryAny Any() { return new QueryAny(); } public static BsonExpression And(BsonExpression left, BsonExpression right) { if (left == null) { throw new ArgumentNullException("left"); } if (right == null) { throw new ArgumentNullException("right"); } return "(" + left.Source + " AND " + right.Source + ")"; } public static BsonExpression And(params BsonExpression[] queries) { if (queries == null || queries.Length < 2) { throw new ArgumentException("At least two Query should be passed"); } BsonExpression bsonExpression = queries[0]; for (int i = 1; i < queries.Length; i++) { bsonExpression = And(bsonExpression, queries[i]); } return bsonExpression; } public static BsonExpression Or(BsonExpression left, BsonExpression right) { if (left == null) { throw new ArgumentNullException("left"); } if (right == null) { throw new ArgumentNullException("right"); } return "(" + left.Source + " OR " + right.Source + ")"; } public static BsonExpression Or(params BsonExpression[] queries) { if (queries == null || queries.Length < 2) { throw new ArgumentException("At least two Query should be passed"); } BsonExpression bsonExpression = queries[0]; for (int i = 1; i < queries.Length; i++) { bsonExpression = Or(bsonExpression, queries[i]); } return bsonExpression; } public string ToSQL(string collection) { StringBuilder stringBuilder = new StringBuilder(); if (ExplainPlan) { stringBuilder.AppendLine("EXPLAIN"); } stringBuilder.AppendLine("SELECT " + Select.Source); if (Into != null) { stringBuilder.AppendLine("INTO " + Into + ":" + IntoAutoId.ToString().ToLower()); } stringBuilder.AppendLine("FROM " + collection); if (Includes.Count > 0) { stringBuilder.AppendLine("INCLUDE " + string.Join(", ", Includes.Select((BsonExpression x) => x.Source))); } if (Where.Count > 0) { stringBuilder.AppendLine("WHERE " + string.Join(" AND ", Where.Select((BsonExpression x) => x.Source))); } if (GroupBy != null) { stringBuilder.AppendLine("GROUP BY " + GroupBy.Source); } if (Having != null) { stringBuilder.AppendLine("HAVING " + Having.Source); } if (OrderBy != null) { stringBuilder.AppendLine("ORDER BY " + OrderBy.Source + " " + ((Order == 1) ? "ASC" : "DESC")); } if (Limit != int.MaxValue) { stringBuilder.AppendLine($"LIMIT {Limit}"); } if (Offset != 0) { stringBuilder.AppendLine($"OFFSET {Offset}"); } if (ForUpdate) { stringBuilder.AppendLine("FOR UPDATE"); } return stringBuilder.ToString().Trim(); } } public class QueryAny { public BsonExpression EQ(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY = {value ?? BsonValue.Null}"); } public BsonExpression LT(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY < {value ?? BsonValue.Null}"); } public BsonExpression LTE(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY <= {value ?? BsonValue.Null}"); } public BsonExpression GT(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY > {value ?? BsonValue.Null}"); } public BsonExpression GTE(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY >= {value ?? BsonValue.Null}"); } public BsonExpression Between(string arrayField, BsonValue start, BsonValue end) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY BETWEEN {start ?? BsonValue.Null} AND {end ?? BsonValue.Null}"); } public BsonExpression StartsWith(string arrayField, string value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } if (value.IsNullOrWhiteSpace()) { throw new ArgumentNullException("value"); } return BsonExpression.Create(string.Format("{0} ANY LIKE {1}", arrayField, new BsonValue(value + "%"))); } public BsonExpression Not(string arrayField, BsonValue value) { if (arrayField.IsNullOrWhiteSpace()) { throw new ArgumentNullException("arrayField"); } return BsonExpression.Create($"{arrayField} ANY != {value ?? BsonValue.Null}"); } } public class BsonArray : BsonValue, IList, ICollection, IEnumerable, IEnumerable { private int _length; public new IList RawValue => (IList)base.RawValue; public override BsonValue this[int index] { get { return RawValue[index]; } set { RawValue[index] = value ?? BsonValue.Null; } } public int Count => RawValue.Count; public bool IsReadOnly => false; public BsonArray() : base(BsonType.Array, new List()) { } public BsonArray(List array) : this() { if (array == null) { throw new ArgumentNullException("array"); } AddRange(array); } public BsonArray(params BsonValue[] array) : this() { if (array == null) { throw new ArgumentNullException("array"); } AddRange(array); } public BsonArray(IEnumerable items) : this() { if (items == null) { throw new ArgumentNullException("items"); } AddRange(items); } public void Add(BsonValue item) { RawValue.Add(item ?? BsonValue.Null); } public void AddRange(TCollection collection) where TCollection : ICollection { if (collection == null) { throw new ArgumentNullException("collection"); } List list = (List)base.RawValue; if (list.Capacity - list.Count < collection.Count) { list.Capacity += collection.Count; } foreach (BsonValue item in collection) { list.Add(item ?? BsonValue.Null); } } public void AddRange(IEnumerable items) { if (items == null) { throw new ArgumentNullException("items"); } foreach (BsonValue item in items) { Add(item ?? BsonValue.Null); } } public void Clear() { RawValue.Clear(); } public bool Contains(BsonValue item) { return RawValue.Contains(item ?? BsonValue.Null); } public void CopyTo(BsonValue[] array, int arrayIndex) { RawValue.CopyTo(array, arrayIndex); } public IEnumerator GetEnumerator() { return RawValue.GetEnumerator(); } public int IndexOf(BsonValue item) { return RawValue.IndexOf(item ?? BsonValue.Null); } public void Insert(int index, BsonValue item) { RawValue.Insert(index, item ?? BsonValue.Null); } public bool Remove(BsonValue item) { return RawValue.Remove(item); } public void RemoveAt(int index) { RawValue.RemoveAt(index); } IEnumerator IEnumerable.GetEnumerator() { foreach (BsonValue item in RawValue) { yield return item; } } public override int CompareTo(BsonValue other) { if (other.Type != BsonType.Array) { return base.Type.CompareTo(other.Type); } BsonArray asArray = other.AsArray; int num = 0; int num2 = 0; int num3 = Math.Min(Count, asArray.Count); while (num == 0 && num2 < num3) { num = this[num2].CompareTo(asArray[num2]); num2++; } if (num != 0) { return num; } if (num2 == Count) { if (num2 != asArray.Count) { return -1; } return 0; } return 1; } internal override int GetBytesCount(bool recalc) { if (!recalc && _length > 0) { return _length; } int num = 5; IList rawValue = RawValue; for (int i = 0; i < rawValue.Count; i++) { num += GetBytesCountElement(i.ToString(), rawValue[i]); } return _length = num; } } public enum BsonAutoId { Int32 = 2, Int64 = 3, ObjectId = 10, Guid = 11 } public class BsonDocument : BsonValue, IDictionary, ICollection>, IEnumerable>, IEnumerable { private int _length; public new IDictionary RawValue => base.RawValue as IDictionary; internal PageAddress RawId { get; set; } = PageAddress.Empty; public override BsonValue this[string key] { get { return RawValue.GetOrDefault(key, BsonValue.Null); } set { RawValue[key] = value ?? BsonValue.Null; } } public ICollection Keys => RawValue.Keys; public ICollection Values => RawValue.Values; public int Count => RawValue.Count; public bool IsReadOnly => false; public BsonDocument() : base(BsonType.Document, new Dictionary(StringComparer.OrdinalIgnoreCase)) { } public BsonDocument(ConcurrentDictionary dict) : this() { if (dict == null) { throw new ArgumentNullException("dict"); } foreach (KeyValuePair item in dict) { Add(item); } } public BsonDocument(IDictionary dict) : this() { if (dict == null) { throw new ArgumentNullException("dict"); } foreach (KeyValuePair item in dict) { Add(item); } } public override int CompareTo(BsonValue other) { if (other.Type != BsonType.Document) { return base.Type.CompareTo(other.Type); } string[] array = Keys.ToArray(); int num = array.Length; BsonDocument asDocument = other.AsDocument; int num2 = asDocument.Keys.ToArray().Length; int num3 = 0; int num4 = 0; int num5 = Math.Min(num, num2); while (num3 == 0 && num4 < num5) { num3 = this[array[num4]].CompareTo(asDocument[array[num4]]); num4++; } if (num3 != 0) { return num3; } if (num4 == num) { if (num4 != num2) { return -1; } return 0; } return 1; } public bool ContainsKey(string key) { return RawValue.ContainsKey(key); } public IEnumerable> GetElements() { if (RawValue.TryGetValue("_id", out var value)) { yield return new KeyValuePair("_id", value); } foreach (KeyValuePair item in RawValue.Where((KeyValuePair x) => x.Key != "_id")) { yield return item; } } public void Add(string key, BsonValue value) { RawValue.Add(key, value ?? BsonValue.Null); } public bool Remove(string key) { return RawValue.Remove(key); } public void Clear() { RawValue.Clear(); } public bool TryGetValue(string key, out BsonValue value) { return RawValue.TryGetValue(key, out value); } public void Add(KeyValuePair item) { Add(item.Key, item.Value); } public bool Contains(KeyValuePair item) { return RawValue.Contains(item); } public bool Remove(KeyValuePair item) { return Remove(item.Key); } public IEnumerator> GetEnumerator() { return RawValue.GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() { return RawValue.GetEnumerator(); } public void CopyTo(KeyValuePair[] array, int arrayIndex) { RawValue.CopyTo(array, arrayIndex); } public void CopyTo(BsonDocument other) { using IEnumerator> enumerator = GetEnumerator(); while (enumerator.MoveNext()) { KeyValuePair current = enumerator.Current; other[current.Key] = current.Value; } } internal override int GetBytesCount(bool recalc) { if (!recalc && _length > 0) { return _length; } int num = 5; foreach (KeyValuePair item in RawValue) { num += GetBytesCountElement(item.Key, item.Value); } return _length = num; } } public enum BsonType : byte { MinValue, Null, Int32, Int64, Double, Decimal, String, Document, Array, Binary, ObjectId, Guid, Boolean, DateTime, MaxValue } public class BsonValue : IComparable, IEquatable { public static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); public static BsonValue Null = new BsonValue(BsonType.Null, null); public static BsonValue MinValue = new BsonValue(BsonType.MinValue, "-oo"); public static BsonValue MaxValue = new BsonValue(BsonType.MaxValue, "+oo"); public BsonType Type { get; } public virtual object RawValue { get; } public virtual BsonValue this[string name] { get { throw new InvalidOperationException("Cannot access non-document type value on " + RawValue); } set { throw new InvalidOperationException("Cannot access non-document type value on " + RawValue); } } public virtual BsonValue this[int index] { get { throw new InvalidOperationException("Cannot access non-array type value on " + RawValue); } set { throw new InvalidOperationException("Cannot access non-array type value on " + RawValue); } } [DebuggerBrowsable(DebuggerBrowsableState.Never)] public BsonArray AsArray => this as BsonArray; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public BsonDocument AsDocument => this as BsonDocument; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public byte[] AsBinary => RawValue as byte[]; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool AsBoolean => (bool)RawValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public string AsString => (string)RawValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public int AsInt32 => Convert.ToInt32(RawValue); [DebuggerBrowsable(DebuggerBrowsableState.Never)] public long AsInt64 => Convert.ToInt64(RawValue); [DebuggerBrowsable(DebuggerBrowsableState.Never)] public double AsDouble => Convert.ToDouble(RawValue); [DebuggerBrowsable(DebuggerBrowsableState.Never)] public decimal AsDecimal => Convert.ToDecimal(RawValue); [DebuggerBrowsable(DebuggerBrowsableState.Never)] public DateTime AsDateTime => (DateTime)RawValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public ObjectId AsObjectId => (ObjectId)RawValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public Guid AsGuid => (Guid)RawValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsNull => Type == BsonType.Null; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsArray => Type == BsonType.Array; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsDocument => Type == BsonType.Document; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsInt32 => Type == BsonType.Int32; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsInt64 => Type == BsonType.Int64; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsDouble => Type == BsonType.Double; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsDecimal => Type == BsonType.Decimal; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsNumber { get { if (!IsInt32 && !IsInt64 && !IsDouble) { return IsDecimal; } return true; } } [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsBinary => Type == BsonType.Binary; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsBoolean => Type == BsonType.Boolean; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsString => Type == BsonType.String; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsObjectId => Type == BsonType.ObjectId; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsGuid => Type == BsonType.Guid; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsDateTime => Type == BsonType.DateTime; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsMinValue => Type == BsonType.MinValue; [DebuggerBrowsable(DebuggerBrowsableState.Never)] public bool IsMaxValue => Type == BsonType.MaxValue; public static BsonDocument DbRef(BsonValue id, string collection) { return new BsonDocument { ["$id"] = id, ["$ref"] = collection }; } public BsonValue() { Type = BsonType.Null; RawValue = null; } public BsonValue(int value) { Type = BsonType.Int32; RawValue = value; } public BsonValue(long value) { Type = BsonType.Int64; RawValue = value; } public BsonValue(double value) { Type = BsonType.Double; RawValue = value; } public BsonValue(decimal value) { Type = BsonType.Decimal; RawValue = value; } public BsonValue(string value) { Type = ((value == null) ? BsonType.Null : BsonType.String); RawValue = value; } public BsonValue(byte[] value) { Type = ((value == null) ? BsonType.Null : BsonType.Binary); RawValue = value; } public BsonValue(ObjectId value) { Type = ((value == null) ? BsonType.Null : BsonType.ObjectId); RawValue = value; } public BsonValue(Guid value) { Type = BsonType.Guid; RawValue = value; } public BsonValue(bool value) { Type = BsonType.Boolean; RawValue = value; } public BsonValue(DateTime value) { Type = BsonType.DateTime; RawValue = value.Truncate(); } protected BsonValue(BsonType type, object rawValue) { Type = type; RawValue = rawValue; } public BsonValue(object value) { RawValue = value; if (value == null) { Type = BsonType.Null; return; } if (value is int) { Type = BsonType.Int32; return; } if (value is long) { Type = BsonType.Int64; return; } if (value is double) { Type = BsonType.Double; return; } if (value is decimal) { Type = BsonType.Decimal; return; } if (value is string) { Type = BsonType.String; return; } if (value is IDictionary) { Type = BsonType.Document; return; } if (value is IList) { Type = BsonType.Array; return; } if (value is byte[]) { Type = BsonType.Binary; return; } if (value is ObjectId) { Type = BsonType.ObjectId; return; } if (value is Guid) { Type = BsonType.Guid; return; } if (value is bool) { Type = BsonType.Boolean; return; } if (value is DateTime) { Type = BsonType.DateTime; RawValue = ((DateTime)value).Truncate(); return; } if (value is BsonValue) { BsonValue bsonValue = (BsonValue)value; Type = bsonValue.Type; RawValue = bsonValue.RawValue; return; } IEnumerable enumerable = value as IEnumerable; if (value is IDictionary dictionary) { Dictionary dictionary2 = new Dictionary(); foreach (object key in dictionary.Keys) { dictionary2.Add(key.ToString(), new BsonValue(dictionary[key])); } Type = BsonType.Document; RawValue = dictionary2; return; } if (enumerable != null) { List list = new List(); foreach (object item in enumerable) { list.Add(new BsonValue(item)); } Type = BsonType.Array; RawValue = list; return; } throw new InvalidCastException("Value is not a valid BSON data type - Use Mapper.ToDocument for more complex types converts"); } public static implicit operator int(BsonValue value) { return (int)value.RawValue; } public static implicit operator BsonValue(int value) { return new BsonValue(value); } public static implicit operator long(BsonValue value) { return (long)value.RawValue; } public static implicit operator BsonValue(long value) { return new BsonValue(value); } public static implicit operator double(BsonValue value) { return (double)value.RawValue; } public static implicit operator BsonValue(double value) { return new BsonValue(value); } public static implicit operator decimal(BsonValue value) { return (decimal)value.RawValue; } public static implicit operator BsonValue(decimal value) { return new BsonValue(value); } public static implicit operator ulong(BsonValue value) { return (ulong)value.RawValue; } public static implicit operator BsonValue(ulong value) { return new BsonValue((double)value); } public static implicit operator string(BsonValue value) { return (string)value.RawValue; } public static implicit operator BsonValue(string value) { return new BsonValue(value); } public static implicit operator byte[](BsonValue value) { return (byte[])value.RawValue; } public static implicit operator BsonValue(byte[] value) { return new BsonValue(value); } public static implicit operator ObjectId(BsonValue value) { return (ObjectId)value.RawValue; } public static implicit operator BsonValue(ObjectId value) { return new BsonValue(value); } public static implicit operator Guid(BsonValue value) { return (Guid)value.RawValue; } public static implicit operator BsonValue(Guid value) { return new BsonValue(value); } public static implicit operator bool(BsonValue value) { return (bool)value.RawValue; } public static implicit operator BsonValue(bool value) { return new BsonValue(value); } public static implicit operator DateTime(BsonValue value) { return (DateTime)value.RawValue; } public static implicit operator BsonValue(DateTime value) { return new BsonValue(value); } public static BsonValue operator +(BsonValue left, BsonValue right) { if (!left.IsNumber || !right.IsNumber) { return Null; } if (left.IsInt32 && right.IsInt32) { return left.AsInt32 + right.AsInt32; } if (left.IsInt64 && right.IsInt64) { return left.AsInt64 + right.AsInt64; } if (left.IsDouble && right.IsDouble) { return left.AsDouble + right.AsDouble; } if (left.IsDecimal && right.IsDecimal) { return left.AsDecimal + right.AsDecimal; } decimal num = left.AsDecimal + right.AsDecimal; return (BsonType)(byte)Math.Max((int)left.Type, (int)right.Type) switch { BsonType.Double => new BsonValue((double)num), BsonType.Int64 => new BsonValue((long)num), _ => new BsonValue(num), }; } public static BsonValue operator -(BsonValue left, BsonValue right) { if (!left.IsNumber || !right.IsNumber) { return Null; } if (left.IsInt32 && right.IsInt32) { return left.AsInt32 - right.AsInt32; } if (left.IsInt64 && right.IsInt64) { return left.AsInt64 - right.AsInt64; } if (left.IsDouble && right.IsDouble) { return left.AsDouble - right.AsDouble; } if (left.IsDecimal && right.IsDecimal) { return left.AsDecimal - right.AsDecimal; } decimal num = left.AsDecimal - right.AsDecimal; return (BsonType)(byte)Math.Max((int)left.Type, (int)right.Type) switch { BsonType.Double => new BsonValue((double)num), BsonType.Int64 => new BsonValue((long)num), _ => new BsonValue(num), }; } public static BsonValue operator *(BsonValue left, BsonValue right) { if (!left.IsNumber || !right.IsNumber) { return Null; } if (left.IsInt32 && right.IsInt32) { return left.AsInt32 * right.AsInt32; } if (left.IsInt64 && right.IsInt64) { return left.AsInt64 * right.AsInt64; } if (left.IsDouble && right.IsDouble) { return left.AsDouble * right.AsDouble; } if (left.IsDecimal && right.IsDecimal) { return left.AsDecimal * right.AsDecimal; } decimal num = left.AsDecimal * right.AsDecimal; return (BsonType)(byte)Math.Max((int)left.Type, (int)right.Type) switch { BsonType.Double => new BsonValue((double)num), BsonType.Int64 => new BsonValue((long)num), _ => new BsonValue(num), }; } public static BsonValue operator /(BsonValue left, BsonValue right) { if (!left.IsNumber || !right.IsNumber) { return Null; } if (left.IsDecimal || right.IsDecimal) { return left.AsDecimal / right.AsDecimal; } return left.AsDouble / right.AsDouble; } public override string ToString() { return JsonSerializer.Serialize(this); } public virtual int CompareTo(BsonValue other) { return CompareTo(other, Collation.Binary); } public virtual int CompareTo(BsonValue other, Collation collation) { if (Type != other.Type) { if (IsNumber && other.IsNumber) { return Convert.ToDecimal(RawValue).CompareTo(Convert.ToDecimal(other.RawValue)); } int num = Type.CompareTo(other.Type); if (num >= 0) { return (num > 0) ? 1 : 0; } return -1; } switch (Type) { case BsonType.MinValue: case BsonType.Null: case BsonType.MaxValue: return 0; case BsonType.Int32: return AsInt32.CompareTo(other.AsInt32); case BsonType.Int64: return AsInt64.CompareTo(other.AsInt64); case BsonType.Double: return AsDouble.CompareTo(other.AsDouble); case BsonType.Decimal: return AsDecimal.CompareTo(other.AsDecimal); case BsonType.String: return collation.Compare(AsString, other.AsString); case BsonType.Document: return AsDocument.CompareTo(other); case BsonType.Array: return AsArray.CompareTo(other); case BsonType.Binary: return AsBinary.BinaryCompareTo(other.AsBinary); case BsonType.ObjectId: return AsObjectId.CompareTo(other.AsObjectId); case BsonType.Guid: return AsGuid.CompareTo(other.AsGuid); case BsonType.Boolean: return AsBoolean.CompareTo(other.AsBoolean); case BsonType.DateTime: { DateTime dateTime = AsDateTime; DateTime value = other.AsDateTime; if (dateTime.Kind != DateTimeKind.Utc) { dateTime = dateTime.ToUniversalTime(); } if (value.Kind != DateTimeKind.Utc) { value = value.ToUniversalTime(); } return dateTime.CompareTo(value); } default: throw new NotImplementedException(); } } public bool Equals(BsonValue other) { return CompareTo(other) == 0; } public static bool operator ==(BsonValue lhs, BsonValue rhs) { if ((object)lhs == null) { return (object)rhs == null; } if ((object)rhs == null) { return false; } return lhs.Equals(rhs); } public static bool operator !=(BsonValue lhs, BsonValue rhs) { return !(lhs == rhs); } public static bool operator >=(BsonValue lhs, BsonValue rhs) { return lhs.CompareTo(rhs) >= 0; } public static bool operator >(BsonValue lhs, BsonValue rhs) { return lhs.CompareTo(rhs) > 0; } public static bool operator <(BsonValue lhs, BsonValue rhs) { return lhs.CompareTo(rhs) < 0; } public static bool operator <=(BsonValue lhs, BsonValue rhs) { return lhs.CompareTo(rhs) <= 0; } public override bool Equals(object obj) { if (obj is BsonValue other) { return Equals(other); } return false; } public override int GetHashCode() { int num = 17; num = 37 * num + Type.GetHashCode(); return 37 * num + (RawValue?.GetHashCode() ?? 0); } internal virtual int GetBytesCount(bool recalc) { switch (Type) { case BsonType.MinValue: case BsonType.Null: case BsonType.MaxValue: return 0; case BsonType.Int32: return 4; case BsonType.Int64: return 8; case BsonType.Double: return 8; case BsonType.Decimal: return 16; case BsonType.String: return StringEncoding.UTF8.GetByteCount(AsString); case BsonType.Binary: return AsBinary.Length; case BsonType.ObjectId: return 12; case BsonType.Guid: return 16; case BsonType.Boolean: return 1; case BsonType.DateTime: return 8; case BsonType.Document: return AsDocument.GetBytesCount(recalc); case BsonType.Array: return AsArray.GetBytesCount(recalc); default: throw new ArgumentException(); } } protected int GetBytesCountElement(string key, BsonValue value) { bool flag = value.Type == BsonType.String || value.Type == BsonType.Binary || value.Type == BsonType.Guid; return 1 + StringEncoding.UTF8.GetByteCount(key) + 1 + value.GetBytesCount(recalc: true) + (flag ? 5 : 0); } } public class BsonSerializer { public static byte[] Serialize(BsonDocument doc) { if (doc == null) { throw new ArgumentNullException("doc"); } byte[] array = new byte[doc.GetBytesCount(recalc: true)]; using BufferWriter bufferWriter = new BufferWriter(array); bufferWriter.WriteDocument(doc, recalc: false); return array; } public static BsonDocument Deserialize(byte[] buffer, bool utcDate = false, HashSet fields = null) { if (buffer == null || buffer.Length == 0) { throw new ArgumentNullException("buffer"); } using BufferReader bufferReader = new BufferReader(buffer, utcDate); return bufferReader.ReadDocument(fields).GetValue(); } } public class BsonDataReader : IBsonDataReader, IDisposable { private readonly IEnumerator _source; private readonly EngineState _state; private readonly string _collection; private readonly bool _hasValues; private BsonValue _current; private bool _isFirst; private bool _disposed; public bool HasValues => _hasValues; public BsonValue Current => _current; public string Collection => _collection; public BsonValue this[string field] => _current.AsDocument[field] ?? BsonValue.Null; internal BsonDataReader() { _hasValues = false; } internal BsonDataReader(BsonValue value, string collection = null) { _current = value; _isFirst = (_hasValues = true); _collection = collection; } internal BsonDataReader(IEnumerable values, string collection, EngineState state) { _collection = collection; _source = values.GetEnumerator(); _state = state; try { _state.Validate(); if (_source.MoveNext()) { _hasValues = (_isFirst = true); _current = _state.ReadTransform(_collection, _source.Current); } } catch (Exception ex) { _state.Handle(ex); throw; } } public bool Read() { if (!_hasValues) { return false; } if (_isFirst) { _isFirst = false; return true; } if (_source != null) { _state.Validate(); try { bool result = _source.MoveNext(); _current = _state.ReadTransform(_collection, _source.Current); return result; } catch (Exception ex) { _state.Handle(ex); throw ex; } } return false; } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } ~BsonDataReader() { Dispose(disposing: false); } protected virtual void Dispose(bool disposing) { if (!_disposed) { _disposed = true; if (disposing) { _source?.Dispose(); } } } } public static class BsonDataReaderExtensions { public static IEnumerable ToEnumerable(this IBsonDataReader reader) { try { while (reader.Read()) { yield return reader.Current; } } finally { reader.Dispose(); } } public static BsonValue[] ToArray(this IBsonDataReader reader) { return reader.ToEnumerable().ToArray(); } public static IList ToList(this IBsonDataReader reader) { return reader.ToEnumerable().ToList(); } public static BsonValue First(this IBsonDataReader reader) { return reader.ToEnumerable().First(); } public static BsonValue FirstOrDefault(this IBsonDataReader reader) { return reader.ToEnumerable().FirstOrDefault(); } public static BsonValue Single(this IBsonDataReader reader) { return reader.ToEnumerable().Single(); } public static BsonValue SingleOrDefault(this IBsonDataReader reader) { return reader.ToEnumerable().SingleOrDefault(); } } public interface IBsonDataReader : IDisposable { BsonValue this[string field] { get; } string Collection { get; } BsonValue Current { get; } bool HasValues { get; } bool Read(); } internal delegate IEnumerable BsonExpressionEnumerableDelegate(IEnumerable source, BsonDocument root, BsonValue current, Collation collation, BsonDocument parameters); internal delegate BsonValue BsonExpressionScalarDelegate(IEnumerable source, BsonDocument root, BsonValue current, Collation collation, BsonDocument parameters); public sealed class BsonExpression { private BsonExpressionEnumerableDelegate _funcEnumerable; private BsonExpressionScalarDelegate _funcScalar; private static readonly ConcurrentDictionary _cacheEnumerable = new ConcurrentDictionary(); private static readonly ConcurrentDictionary _cacheScalar = new ConcurrentDictionary(); public static BsonExpression Root = Create("$"); private static readonly Dictionary _methods = typeof(BsonExpressionMethods).GetMethods(BindingFlags.Static | BindingFlags.Public).ToDictionary((MethodInfo m) => m.Name.ToUpperInvariant() + "~" + (from p in m.GetParameters() where p.ParameterType != typeof(Collation) select p).Count()); private static readonly Dictionary _functions = typeof(BsonExpressionFunctions).GetMethods(BindingFlags.Static | BindingFlags.Public).ToDictionary((MethodInfo m) => m.Name.ToUpperInvariant() + "~" + m.GetParameters().Skip(5).Count()); public string Source { get; internal set; } public BsonExpressionType Type { get; internal set; } public bool IsImmutable { get; internal set; } public BsonDocument Parameters { get; internal set; } internal BsonExpression Left { get; set; } internal BsonExpression Right { get; set; } internal bool UseSource { get; set; } internal Expression Expression { get; set; } public HashSet Fields { get; internal set; } public bool IsScalar { get; internal set; } internal bool IsPredicate { get { if (Type != BsonExpressionType.Equal && Type != BsonExpressionType.Like && Type != BsonExpressionType.Between && Type != BsonExpressionType.GreaterThan && Type != BsonExpressionType.GreaterThanOrEqual && Type != BsonExpressionType.LessThan && Type != BsonExpressionType.LessThanOrEqual && Type != BsonExpressionType.NotEqual) { return Type == BsonExpressionType.In; } return true; } } internal bool IsIndexable { get { if (Fields.Count > 0 && IsImmutable) { return Parameters.Count == 0; } return false; } } internal bool IsValue => Fields.Count == 0; internal bool IsANY { get { if (IsPredicate) { return Expression.ToString().Contains("_ANY"); } return false; } } public static IEnumerable Methods => _methods.Values; public static IEnumerable Functions => _functions.Values; internal string DefaultFieldName() { string text = string.Join("_", Fields.Where((string x) => x != "$")); if (!string.IsNullOrEmpty(text)) { return text; } return "expr"; } internal BsonExpression() { } public static implicit operator string(BsonExpression expr) { return expr.Source; } public static implicit operator BsonExpression(string expr) { return Create(expr); } public IEnumerable Execute(Collation collation = null) { BsonDocument bsonDocument = new BsonDocument(); BsonDocument[] source = new BsonDocument[1] { bsonDocument }; return Execute(source, bsonDocument, bsonDocument, collation); } public IEnumerable Execute(BsonDocument root, Collation collation = null) { if (root == null) { throw new ArgumentNullException("root"); } BsonDocument[] source = new BsonDocument[1] { root }; return Execute(source, root, root, collation); } public IEnumerable Execute(IEnumerable source, Collation collation = null) { if (source == null) { throw new ArgumentNullException("source"); } return Execute(source, null, null, collation); } internal IEnumerable Execute(IEnumerable source, BsonDocument root, BsonValue current, Collation collation) { if (IsScalar) { yield return _funcScalar(source, root, current, collation ?? Collation.Binary, Parameters); yield break; } IEnumerable enumerable = _funcEnumerable(source, root, current, collation ?? Collation.Binary, Parameters); foreach (BsonValue item in enumerable) { yield return item; } } internal IEnumerable GetIndexKeys(BsonDocument doc, Collation collation) { return Execute(doc, collation).Distinct(); } public BsonValue ExecuteScalar(Collation collation = null) { BsonDocument bsonDocument = new BsonDocument(); BsonDocument[] source = new BsonDocument[0]; return ExecuteScalar(source, bsonDocument, bsonDocument, collation); } public BsonValue ExecuteScalar(BsonDocument root, Collation collation = null) { if (root == null) { throw new ArgumentNullException("root"); } BsonDocument[] source = new BsonDocument[1] { root }; return ExecuteScalar(source, root, root, collation); } public BsonValue ExecuteScalar(IEnumerable source, Collation collation = null) { if (source == null) { throw new ArgumentNullException("source"); } return ExecuteScalar(source, null, null, collation); } internal BsonValue ExecuteScalar(IEnumerable source, BsonDocument root, BsonValue current, Collation collation) { if (IsScalar) { return _funcScalar(source, root, current, collation ?? Collation.Binary, Parameters); } throw new LiteException(0, "Expression `" + Source + "` is not a scalar expression and can return more than one result"); } public static BsonExpression Create(string expression) { return Create(expression, new BsonDocument()); } public static BsonExpression Create(string expression, params BsonValue[] args) { BsonDocument bsonDocument = new BsonDocument(); for (int i = 0; i < args.Length; i++) { bsonDocument[i.ToString()] = args[i]; } return Create(expression, bsonDocument); } public static BsonExpression Create(string expression, BsonDocument parameters) { if (string.IsNullOrWhiteSpace(expression)) { throw new ArgumentNullException("expression"); } Tokenizer tokenizer = new Tokenizer(expression); BsonExpression result = Create(tokenizer, BsonExpressionParserMode.Full, parameters); tokenizer.LookAhead().Expect(TokenType.EOF); return result; } internal static BsonExpression Create(Tokenizer tokenizer, BsonExpressionParserMode mode, BsonDocument parameters) { if (tokenizer == null) { throw new ArgumentNullException("tokenizer"); } return ParseAndCompile(tokenizer, mode, parameters, DocumentScope.Root); } internal static BsonExpression ParseAndCompile(Tokenizer tokenizer, BsonExpressionParserMode mode, BsonDocument parameters, DocumentScope scope) { if (tokenizer == null) { throw new ArgumentNullException("tokenizer"); } ExpressionContext context = new ExpressionContext(); object obj = mode switch { BsonExpressionParserMode.SelectDocument => BsonExpressionParser.ParseSelectDocumentBuilder(tokenizer, context, parameters), BsonExpressionParserMode.Single => BsonExpressionParser.ParseSingleExpression(tokenizer, context, parameters, scope), BsonExpressionParserMode.Full => BsonExpressionParser.ParseFullExpression(tokenizer, context, parameters, scope), _ => BsonExpressionParser.ParseUpdateDocumentBuilder(tokenizer, context, parameters), }; Compile((BsonExpression)obj, context); return (BsonExpression)obj; } internal static void Compile(BsonExpression expr, ExpressionContext context) { if (expr.IsScalar) { BsonExpressionScalarDelegate orAdd = _cacheScalar.GetOrAdd(expr.Source, (string s) => Expression.Lambda(expr.Expression, new ParameterExpression[5] { context.Source, context.Root, context.Current, context.Collation, context.Parameters }).Compile()); expr._funcScalar = orAdd; } else { BsonExpressionEnumerableDelegate orAdd2 = _cacheEnumerable.GetOrAdd(expr.Source, (string s) => Expression.Lambda(expr.Expression, new ParameterExpression[5] { context.Source, context.Root, context.Current, context.Collation, context.Parameters }).Compile()); expr._funcEnumerable = orAdd2; } if (expr.Left != null) { Compile(expr.Left, context); } if (expr.Right != null) { Compile(expr.Right, context); } } internal static void SetParameters(BsonExpression expr, BsonDocument parameters) { expr.Parameters = parameters; if (expr.Left != null) { SetParameters(expr.Left, parameters); } if (expr.Right != null) { SetParameters(expr.Right, parameters); } } internal static MethodInfo GetMethod(string name, int parameterCount) { string key = name.ToUpperInvariant() + "~" + parameterCount; return _methods.GetOrDefault(key); } internal static MethodInfo GetFunction(string name, int parameterCount = 0) { string key = name.ToUpperInvariant() + "~" + parameterCount; return _functions.GetOrDefault(key); } public override string ToString() { return $"`{Source}` [{Type}]"; } } internal class BsonExpressionMethods { private static readonly Random _random = new Random(); public static BsonValue COUNT(IEnumerable values) { return values.Count(); } public static BsonValue MIN(IEnumerable values) { BsonValue bsonValue = BsonValue.MaxValue; foreach (BsonValue value in values) { if (value.CompareTo(bsonValue) <= 0) { bsonValue = value; } } if (!(bsonValue == BsonValue.MaxValue)) { return bsonValue; } return BsonValue.MinValue; } public static BsonValue MAX(IEnumerable values) { BsonValue bsonValue = BsonValue.MinValue; foreach (BsonValue value in values) { if (value.CompareTo(bsonValue) >= 0) { bsonValue = value; } } if (!(bsonValue == BsonValue.MinValue)) { return bsonValue; } return BsonValue.MaxValue; } public static BsonValue FIRST(IEnumerable values) { return values.FirstOrDefault(); } public static BsonValue LAST(IEnumerable values) { return values.LastOrDefault(); } public static BsonValue AVG(IEnumerable values) { BsonValue bsonValue = new BsonValue(0); int num = 0; foreach (BsonValue item in values.Where((BsonValue x) => x.IsNumber)) { bsonValue += item; num++; } if (num > 0) { return bsonValue / num; } return 0; } public static BsonValue SUM(IEnumerable values) { BsonValue result = new BsonValue(0); foreach (BsonValue item in values.Where((BsonValue x) => x.IsNumber)) { result += item; } return result; } public static BsonValue ANY(IEnumerable values) { return values.Any(); } public static BsonValue MINVALUE() { return BsonValue.MinValue; } [Volatile] public static BsonValue OBJECTID() { return ObjectId.NewObjectId(); } [Volatile] public static BsonValue GUID() { return Guid.NewGuid(); } [Volatile] public static BsonValue NOW() { return DateTime.Now; } [Volatile] public static BsonValue NOW_UTC() { return DateTime.UtcNow; } [Volatile] public static BsonValue TODAY() { return DateTime.Today; } public static BsonValue MAXVALUE() { return BsonValue.MaxValue; } public static BsonValue INT32(BsonValue value) { if (value.IsNumber) { return value.AsInt32; } if (value.IsString && int.TryParse(value.AsString, out var result)) { return result; } return BsonValue.Null; } public static BsonValue INT64(BsonValue value) { if (value.IsNumber) { return value.AsInt64; } if (value.IsString && long.TryParse(value.AsString, out var result)) { return result; } return BsonValue.Null; } public static BsonValue DOUBLE(Collation collation, BsonValue value) { if (value.IsNumber) { return value.AsDouble; } if (value.IsString && double.TryParse(value.AsString, NumberStyles.Any, collation.Culture.NumberFormat, out var result)) { return result; } return BsonValue.Null; } public static BsonValue DOUBLE(BsonValue value, BsonValue culture) { if (value.IsNumber) { return value.AsDouble; } if (value.IsString && culture.IsString) { CultureInfo cultureInfo = new CultureInfo(culture.AsString); if (double.TryParse(value.AsString, NumberStyles.Any, cultureInfo.NumberFormat, out var result)) { return result; } } return BsonValue.Null; } public static BsonValue DECIMAL(Collation collation, BsonValue value) { if (value.IsNumber) { return value.AsDecimal; } if (value.IsString && decimal.TryParse(value.AsString, NumberStyles.Any, collation.Culture.NumberFormat, out var result)) { return result; } return BsonValue.Null; } public static BsonValue DECIMAL(BsonValue value, BsonValue culture) { if (value.IsNumber) { return value.AsDecimal; } if (value.IsString && culture.IsString) { CultureInfo cultureInfo = new CultureInfo(culture.AsString); if (decimal.TryParse(value.AsString, NumberStyles.Any, cultureInfo.NumberFormat, out var result)) { return result; } } return BsonValue.Null; } public static BsonValue STRING(BsonValue value) { return value.IsNull ? "" : (value.IsString ? value.AsString : value.ToString()); } public static BsonValue ARRAY(IEnumerable values) { return new BsonArray(values); } public static BsonValue BINARY(BsonValue value) { if (value.IsBinary) { return value; } if (value.IsString) { byte[] array = null; bool flag = false; try { array = Convert.FromBase64String(value.AsString); flag = true; } catch (FormatException) { } if (flag) { return array; } } return BsonValue.Null; } public static BsonValue OBJECTID(BsonValue value) { if (value.IsObjectId) { return value.AsObjectId; } if (value.IsString) { ObjectId objectId = null; bool flag = false; try { objectId = new ObjectId(value.AsString); flag = true; } catch { } if (flag) { return objectId; } } return BsonValue.Null; } public static BsonValue GUID(BsonValue value) { if (value.IsGuid) { return value.AsGuid; } if (value.IsString) { Guid guid = Guid.Empty; bool flag = false; try { guid = new Guid(value.AsString); flag = true; } catch { } if (flag) { return guid; } } return BsonValue.Null; } public static BsonValue BOOLEAN(BsonValue value) { if (value.IsBoolean) { return value.AsBoolean; } bool flag = false; bool flag2 = false; try { flag = Convert.ToBoolean(value.AsString); flag2 = true; } catch { } if (flag2) { return flag; } return BsonValue.Null; } public static BsonValue DATETIME(Collation collation, BsonValue value) { if (value.IsDateTime) { return value.AsDateTime; } if (value.IsString && DateTime.TryParse(value.AsString, collation.Culture.DateTimeFormat, DateTimeStyles.None, out var result)) { return result; } return BsonValue.Null; } public static BsonValue DATETIME(BsonValue value, BsonValue culture) { if (value.IsDateTime) { return value.AsDateTime; } if (value.IsString && culture.IsString) { CultureInfo cultureInfo = new CultureInfo(culture.AsString); if (DateTime.TryParse(value.AsString, cultureInfo.DateTimeFormat, DateTimeStyles.None, out var result)) { return result; } } return BsonValue.Null; } public static BsonValue DATETIME_UTC(Collation collation, BsonValue value) { if (value.IsDateTime) { return value.AsDateTime; } if (value.IsString && DateTime.TryParse(value.AsString, collation.Culture.DateTimeFormat, DateTimeStyles.AssumeUniversal, out var result)) { return result; } return BsonValue.Null; } public static BsonValue DATETIME_UTC(BsonValue value, BsonValue culture) { if (value.IsDateTime) { return value.AsDateTime; } if (value.IsString && culture.IsString) { CultureInfo cultureInfo = new CultureInfo(culture.AsString); if (DateTime.TryParse(value.AsString, cultureInfo.DateTimeFormat, DateTimeStyles.AssumeUniversal, out var result)) { return result; } } return BsonValue.Null; } public static BsonValue DATETIME(BsonValue year, BsonValue month, BsonValue day) { if (year.IsNumber && month.IsNumber && day.IsNumber) { return new DateTime(year.AsInt32, month.AsInt32, day.AsInt32); } return BsonValue.Null; } public static BsonValue DATETIME_UTC(BsonValue year, BsonValue month, BsonValue day) { if (year.IsNumber && month.IsNumber && day.IsNumber) { return new DateTime(year.AsInt32, month.AsInt32, day.AsInt32, 0, 0, 0, DateTimeKind.Utc); } return BsonValue.Null; } public static BsonValue IS_MINVALUE(BsonValue value) { return value.IsMinValue; } public static BsonValue IS_NULL(BsonValue value) { return value.IsNull; } public static BsonValue IS_INT32(BsonValue value) { return value.IsInt32; } public static BsonValue IS_INT64(BsonValue value) { return value.IsInt64; } public static BsonValue IS_DOUBLE(BsonValue value) { return value.IsDouble; } public static BsonValue IS_DECIMAL(BsonValue value) { return value.IsDecimal; } public static BsonValue IS_NUMBER(BsonValue value) { return value.IsNumber; } public static BsonValue IS_STRING(BsonValue value) { return value.IsString; } public static BsonValue IS_DOCUMENT(BsonValue value) { return value.IsDocument; } public static BsonValue IS_ARRAY(BsonValue value) { return value.IsArray; } public static BsonValue IS_BINARY(BsonValue value) { return value.IsBinary; } public static BsonValue IS_OBJECTID(BsonValue value) { return value.IsObjectId; } public static BsonValue IS_GUID(BsonValue value) { return value.IsGuid; } public static BsonValue IS_BOOLEAN(BsonValue value) { return value.IsBoolean; } public static BsonValue IS_DATETIME(BsonValue value) { return value.IsDateTime; } public static BsonValue IS_MAXVALUE(BsonValue value) { return value.IsMaxValue; } public static BsonValue INT(BsonValue value) { return INT32(value); } public static BsonValue LONG(BsonValue value) { return INT64(value); } public static BsonValue BOOL(BsonValue value) { return BOOLEAN(value); } public static BsonValue DATE(Collation collation, BsonValue value) { return DATETIME(collation, value); } public static BsonValue DATE(BsonValue values, BsonValue culture) { return DATETIME(values, culture); } public static BsonValue DATE_UTC(Collation collation, BsonValue value) { return DATETIME_UTC(collation, value); } public static BsonValue DATE_UTC(BsonValue values, BsonValue culture) { return DATETIME_UTC(values, culture); } public static BsonValue DATE(BsonValue year, BsonValue month, BsonValue day) { return DATETIME(year, month, day); } public static BsonValue DATE_UTC(BsonValue year, BsonValue month, BsonValue day) { return DATETIME_UTC(year, month, day); } public static BsonValue IS_INT(BsonValue value) { return IS_INT32(value); } public static BsonValue IS_LONG(BsonValue value) { return IS_INT64(value); } public static BsonValue IS_BOOL(BsonValue value) { return IS_BOOLEAN(value); } public static BsonValue IS_DATE(BsonValue value) { return IS_DATETIME(value); } public static BsonValue YEAR(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Year; } return BsonValue.Null; } public static BsonValue MONTH(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Month; } return BsonValue.Null; } public static BsonValue DAY(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Day; } return BsonValue.Null; } public static BsonValue HOUR(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Hour; } return BsonValue.Null; } public static BsonValue MINUTE(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Minute; } return BsonValue.Null; } public static BsonValue SECOND(BsonValue value) { if (value.IsDateTime) { return value.AsDateTime.Second; } return BsonValue.Null; } public static BsonValue DATEADD(BsonValue dateInterval, BsonValue number, BsonValue value) { if (dateInterval.IsString && number.IsNumber && value.IsDateTime) { string asString = dateInterval.AsString; int asInt = number.AsInt32; DateTime asDateTime = value.AsDateTime; switch ((asString == "M") ? "month" : asString.ToLower()) { case "y": case "year": return asDateTime.AddYears(asInt); case "month": return asDateTime.AddMonths(asInt); case "d": case "day": return asDateTime.AddDays(asInt); case "h": case "hour": return asDateTime.AddHours(asInt); case "m": case "minute": return asDateTime.AddMinutes(asInt); case "s": case "second": return asDateTime.AddSeconds(asInt); } } return BsonValue.Null; } public static BsonValue DATEDIFF(BsonValue dateInterval, BsonValue starts, BsonValue ends) { if (dateInterval.IsString && starts.IsDateTime && ends.IsDateTime) { string asString = dateInterval.AsString; DateTime asDateTime = starts.AsDateTime; DateTime asDateTime2 = ends.AsDateTime; switch ((asString == "M") ? "month" : asString.ToLower()) { case "y": case "year": return asDateTime.YearDifference(asDateTime2); case "month": return asDateTime.MonthDifference(asDateTime2); case "d": case "day": return Convert.ToInt32(Math.Truncate(asDateTime2.Subtract(asDateTime).TotalDays)); case "h": case "hour": return Convert.ToInt32(Math.Truncate(asDateTime2.Subtract(asDateTime).TotalHours)); case "m": case "minute": return Convert.ToInt32(Math.Truncate(asDateTime2.Subtract(asDateTime).TotalMinutes)); case "s": case "second": return Convert.ToInt32(Math.Truncate(asDateTime2.Subtract(asDateTime).TotalSeconds)); } } return BsonValue.Null; } public static BsonValue TO_LOCAL(BsonValue date) { if (date.IsDateTime) { return date.AsDateTime.ToLocalTime(); } return BsonValue.Null; } public static BsonValue TO_UTC(BsonValue date) { if (date.IsDateTime) { return date.AsDateTime.ToUniversalTime(); } return BsonValue.Null; } public static BsonValue ABS(BsonValue value) { return value.Type switch { BsonType.Int32 => Math.Abs(value.AsInt32), BsonType.Int64 => Math.Abs(value.AsInt64), BsonType.Double => Math.Abs(value.AsDouble), BsonType.Decimal => Math.Abs(value.AsDecimal), _ => BsonValue.Null, }; } public static BsonValue ROUND(BsonValue value, BsonValue digits) { if (digits.IsNumber) { switch (value.Type) { case BsonType.Int32: return value.AsInt32; case BsonType.Int64: return value.AsInt64; case BsonType.Double: return Math.Round(value.AsDouble, digits.AsInt32); case BsonType.Decimal: return Math.Round(value.AsDecimal, digits.AsInt32); } } return BsonValue.Null; } public static BsonValue POW(BsonValue x, BsonValue y) { if (x.IsNumber && y.IsNumber) { return Math.Pow(x.AsDouble, y.AsDouble); } return BsonValue.Null; } public static BsonValue JSON(BsonValue json) { if (json.IsString) { BsonValue result = null; bool flag = false; try { result = JsonSerializer.Deserialize(json.AsString); flag = true; } catch (LiteException ex) when (ex.ErrorCode == 203) { } if (flag) { return result; } } return BsonValue.Null; } public static BsonValue EXTEND(BsonValue source, BsonValue extend) { if (source.IsDocument && extend.IsDocument) { BsonDocument bsonDocument = new BsonDocument(); source.AsDocument.CopyTo(bsonDocument); extend.AsDocument.CopyTo(bsonDocument); bsonDocument.RawId = source.AsDocument.RawId; return bsonDocument; } if (source.IsDocument) { return source; } if (extend.IsDocument) { return extend; } return new BsonDocument(); } public static IEnumerable ITEMS(BsonValue array) { if (array.IsArray) { foreach (BsonValue item in array.AsArray) { yield return item; } } else if (array.IsBinary) { byte[] asBinary = array.AsBinary; foreach (byte b in asBinary) { yield return (int)b; } } else { yield return array; } } public static IEnumerable CONCAT(IEnumerable first, IEnumerable second) { return first.Concat(second); } public static IEnumerable KEYS(BsonValue document) { if (!document.IsDocument) { yield break; } foreach (string key in document.AsDocument.Keys) { yield return key; } } public static IEnumerable VALUES(BsonValue document) { if (!document.IsDocument) { yield break; } foreach (BsonValue value in document.AsDocument.Values) { yield return value; } } public static BsonValue OID_CREATIONTIME(BsonValue objectId) { if (objectId.IsObjectId) { return objectId.AsObjectId.CreationTime; } return BsonValue.Null; } public static BsonValue IIF(BsonValue test, BsonValue ifTrue, BsonValue ifFalse) { throw new NotImplementedException(); } public static BsonValue COALESCE(BsonValue left, BsonValue right) { if (!left.IsNull) { return left; } return right; } public static BsonValue LENGTH(BsonValue value) { if (value.IsString) { return value.AsString.Length; } if (value.IsBinary) { return value.AsBinary.Length; } if (value.IsArray) { return value.AsArray.Count; } if (value.IsDocument) { return value.AsDocument.Keys.Count; } if (value.IsNull) { return 0; } return BsonValue.Null; } public static IEnumerable TOP(IEnumerable values, BsonValue num) { if (num.IsInt32 || num.IsInt64) { int asInt = num.AsInt32; if (asInt > 0) { return values.Take(asInt); } } return Enumerable.Empty(); } public static IEnumerable UNION(IEnumerable left, IEnumerable right) { return left.Union(right); } public static IEnumerable EXCEPT(IEnumerable left, IEnumerable right) { return left.Except(right); } public static IEnumerable DISTINCT(IEnumerable items) { return items.Distinct(); } [Volatile] public static BsonValue RANDOM() { return _random.Next(); } [Volatile] public static BsonValue RANDOM(BsonValue min, BsonValue max) { if (min.IsNumber && max.IsNumber) { return _random.Next(min.AsInt32, max.AsInt32); } return BsonValue.Null; } public static BsonValue LOWER(BsonValue value) { if (value.IsString) { return value.AsString.ToLowerInvariant(); } return BsonValue.Null; } public static BsonValue UPPER(BsonValue value) { if (value.IsString) { return value.AsString.ToUpperInvariant(); } return BsonValue.Null; } public static BsonValue LTRIM(BsonValue value) { if (value.IsString) { return value.AsString.TrimStart(Array.Empty()); } return BsonValue.Null; } public static BsonValue RTRIM(BsonValue value) { if (value.IsString) { return value.AsString.TrimEnd(Array.Empty()); } return BsonValue.Null; } public static BsonValue TRIM(BsonValue value) { if (value.IsString) { return value.AsString.Trim(); } return BsonValue.Null; } public static BsonValue INDEXOF(BsonValue value, BsonValue search) { if (value.IsString && search.IsString) { return value.AsString.IndexOf(search.AsString); } return BsonValue.Null; } public static BsonValue INDEXOF(BsonValue value, BsonValue search, BsonValue startIndex) { if (value.IsString && search.IsString && startIndex.IsNumber) { return value.AsString.IndexOf(search.AsString, startIndex.AsInt32); } return BsonValue.Null; } public static BsonValue SUBSTRING(BsonValue value, BsonValue startIndex) { if (value.IsString && startIndex.IsNumber) { return value.AsString.Substring(startIndex.AsInt32); } return BsonValue.Null; } public static BsonValue SUBSTRING(BsonValue value, BsonValue startIndex, BsonValue length) { if (value.IsString && startIndex.IsNumber && length.IsNumber) { return value.AsString.Substring(startIndex.AsInt32, length.AsInt32); } return BsonValue.Null; } public static BsonValue REPLACE(BsonValue value, BsonValue oldValue, BsonValue newValue) { if (value.IsString && oldValue.IsString && newValue.IsString) { return value.AsString.Replace(oldValue.AsString, newValue.AsString); } return BsonValue.Null; } public static BsonValue LPAD(BsonValue value, BsonValue totalWidth, BsonValue paddingChar) { if (value.IsString && totalWidth.IsNumber && paddingChar.IsString) { string asString = paddingChar.AsString; if (string.IsNullOrEmpty(asString)) { throw new ArgumentOutOfRangeException("paddingChar"); } return value.AsString.PadLeft(totalWidth.AsInt32, asString[0]); } return BsonValue.Null; } public static BsonValue RPAD(BsonValue value, BsonValue totalWidth, BsonValue paddingChar) { if (value.IsString && totalWidth.IsNumber && paddingChar.IsString) { string asString = paddingChar.AsString; if (string.IsNullOrEmpty(asString)) { throw new ArgumentOutOfRangeException("paddingChar"); } return value.AsString.PadRight(totalWidth.AsInt32, asString[0]); } return BsonValue.Null; } public static IEnumerable SPLIT(BsonValue value, BsonValue separator) { if (value.IsString && separator.IsString) { string[] array = value.AsString.Split(new string[1] { separator.AsString }, StringSplitOptions.RemoveEmptyEntries); string[] array2 = array; foreach (string text in array2) { yield return text; } } } public static IEnumerable SPLIT(BsonValue value, BsonValue pattern, BsonValue useRegex) { if (useRegex.IsBoolean && useRegex.AsBoolean) { if (value.IsString && pattern.IsString) { string[] array = Regex.Split(value.AsString, pattern.AsString, RegexOptions.Compiled); string[] array2 = array; foreach (string text in array2) { yield return text; } } yield break; } foreach (BsonValue item in SPLIT(value, pattern)) { yield return item; } } public static BsonValue FORMAT(BsonValue value, BsonValue format) { if (format.IsString) { return string.Format("{0:" + format.AsString + "}", value.RawValue); } return BsonValue.Null; } public static BsonValue JOIN(IEnumerable values) { return JOIN(values, ""); } public static BsonValue JOIN(IEnumerable values, BsonValue separator) { if (separator.IsString) { return string.Join(separator.AsString, values.Select((BsonValue x) => STRING(x).AsString).ToArray()); } return BsonValue.Null; } public static BsonValue IS_MATCH(BsonValue value, BsonValue pattern) { if (!value.IsString || !pattern.IsString) { return false; } return Regex.IsMatch(value.AsString, pattern.AsString); } public static BsonValue MATCH(BsonValue value, BsonValue pattern, BsonValue group) { if (!value.IsString || !pattern.IsString) { return null; } Match match = Regex.Match(value.AsString, pattern.AsString); if (!match.Success) { return BsonValue.Null; } if (group.IsNumber) { return match.Groups[group.AsInt32].Value; } if (group.IsString) { return match.Groups[group.AsString].Value; } return BsonValue.Null; } } internal class VolatileAttribute : Attribute { } internal class BsonExpressionFunctions { public static IEnumerable MAP(BsonDocument root, Collation collation, BsonDocument parameters, IEnumerable input, BsonExpression mapExpr) { foreach (BsonValue item in input) { IEnumerable enumerable = mapExpr.Execute(new BsonDocument[1] { root }, root, item, collation); foreach (BsonValue item2 in enumerable) { yield return item2; } } } public static IEnumerable FILTER(BsonDocument root, Collation collation, BsonDocument parameters, IEnumerable input, BsonExpression filterExpr) { foreach (BsonValue item in input) { BsonValue bsonValue = filterExpr.ExecuteScalar(new BsonDocument[1] { root }, root, item, collation); if (bsonValue.IsBoolean && bsonValue.AsBoolean) { yield return item; } } } public static IEnumerable SORT(BsonDocument root, Collation collation, BsonDocument parameters, IEnumerable input, BsonExpression sortExpr, BsonValue order) { if ((!order.IsInt32 || order.AsInt32 <= 0) && (!order.IsString || !order.AsString.Equals("asc", StringComparison.OrdinalIgnoreCase))) { return from x in source().OrderByDescending((Tuple x) => x.Item2, collation) select x.Item1; } return from x in source().OrderBy((Tuple x) => x.Item2, collation) select x.Item1; IEnumerable> source() { foreach (BsonValue item2 in input) { BsonValue item = sortExpr.ExecuteScalar(new BsonDocument[1] { root }, root, item2, collation); yield return new Tuple(item2, item); } } } public static IEnumerable SORT(BsonDocument root, Collation collation, BsonDocument parameters, IEnumerable input, BsonExpression sortExpr) { return SORT(root, collation, parameters, input, sortExpr, 1); } } internal class BsonExpressionOperators { public static BsonValue ADD(BsonValue left, BsonValue right) { if (left.IsString && right.IsString) { return left.AsString + right.AsString; } if (left.IsString || right.IsString) { return BsonExpressionMethods.STRING(left).AsString + BsonExpressionMethods.STRING(right).AsString; } if (left.IsDateTime && right.IsNumber) { return left.AsDateTime.AddTicks(right.AsInt64); } if (left.IsNumber && right.IsDateTime) { return right.AsDateTime.AddTicks(left.AsInt64); } if (left.IsNumber && right.IsNumber) { return left + right; } return BsonValue.Null; } public static BsonValue MINUS(BsonValue left, BsonValue right) { if (left.IsDateTime && right.IsNumber) { return left.AsDateTime.AddTicks(-right.AsInt64); } if (left.IsNumber && right.IsDateTime) { return right.AsDateTime.AddTicks(-left.AsInt64); } if (left.IsNumber && right.IsNumber) { return left - right; } return BsonValue.Null; } public static BsonValue MULTIPLY(BsonValue left, BsonValue right) { if (left.IsNumber && right.IsNumber) { return left * right; } return BsonValue.Null; } public static BsonValue DIVIDE(BsonValue left, BsonValue right) { if (left.IsNumber && right.IsNumber) { return left / right; } return BsonValue.Null; } public static BsonValue MOD(BsonValue left, BsonValue right) { if (left.IsNumber && right.IsNumber) { return (int)left % (int)right; } return BsonValue.Null; } public static BsonValue EQ(Collation collation, BsonValue left, BsonValue right) { return collation.Equals(left, right); } public static BsonValue EQ_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => collation.Equals(x, right)); } public static BsonValue EQ_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => collation.Equals(x, right)); } public static BsonValue GT(BsonValue left, BsonValue right) { return left > right; } public static BsonValue GT_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => collation.Compare(x, right) > 0); } public static BsonValue GT_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => collation.Compare(x, right) > 0); } public static BsonValue GTE(BsonValue left, BsonValue right) { return left >= right; } public static BsonValue GTE_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => collation.Compare(x, right) >= 0); } public static BsonValue GTE_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => collation.Compare(x, right) >= 0); } public static BsonValue LT(BsonValue left, BsonValue right) { return left < right; } public static BsonValue LT_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => collation.Compare(x, right) < 0); } public static BsonValue LT_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => collation.Compare(x, right) < 0); } public static BsonValue LTE(Collation collation, BsonValue left, BsonValue right) { return left <= right; } public static BsonValue LTE_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => collation.Compare(x, right) <= 0); } public static BsonValue LTE_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => collation.Compare(x, right) <= 0); } public static BsonValue NEQ(Collation collation, BsonValue left, BsonValue right) { return !collation.Equals(left, right); } public static BsonValue NEQ_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => !collation.Equals(x, right)); } public static BsonValue NEQ_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => !collation.Equals(x, right)); } public static BsonValue LIKE(Collation collation, BsonValue left, BsonValue right) { if (left.IsString && right.IsString) { return left.AsString.SqlLike(right.AsString, collation); } return false; } public static BsonValue LIKE_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => LIKE(collation, x, right)); } public static BsonValue LIKE_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => LIKE(collation, x, right)); } public static BsonValue BETWEEN(Collation collation, BsonValue left, BsonValue right) { if (!right.IsArray) { throw new InvalidOperationException("BETWEEN expression need an array with 2 values"); } BsonArray asArray = right.AsArray; if (asArray.Count != 2) { throw new InvalidOperationException("BETWEEN expression need an array with 2 values"); } BsonValue rigth = asArray[0]; BsonValue rigth2 = asArray[1]; return collation.Compare(left, rigth) >= 0 && collation.Compare(left, rigth2) <= 0; } public static BsonValue BETWEEN_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => BETWEEN(collation, x, right)); } public static BsonValue BETWEEN_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => BETWEEN(collation, x, right)); } public static BsonValue IN(Collation collation, BsonValue left, BsonValue right) { if (right.IsArray) { return right.AsArray.Contains(left, collation); } return left == right; } public static BsonValue IN_ANY(Collation collation, IEnumerable left, BsonValue right) { return left.Any((BsonValue x) => IN(collation, x, right)); } public static BsonValue IN_ALL(Collation collation, IEnumerable left, BsonValue right) { return left.All((BsonValue x) => IN(collation, x, right)); } public static BsonValue PARAMETER_PATH(BsonDocument doc, string name) { if (string.IsNullOrEmpty(name)) { return doc; } if (doc.TryGetValue(name, out var value)) { return value; } return BsonValue.Null; } public static BsonValue MEMBER_PATH(BsonValue value, string name) { if (value == null) { throw new LiteException(0, "Field '" + name + "' is invalid in the select list because it is not contained in either an aggregate function or the GROUP BY clause."); } if (string.IsNullOrEmpty(name)) { return value; } if (value.IsDocument && value.AsDocument.TryGetValue(name, out var value2)) { return value2; } return BsonValue.Null; } public static BsonValue ARRAY_INDEX(BsonValue value, int index, BsonExpression expr, BsonDocument root, Collation collation, BsonDocument parameters) { if (!value.IsArray) { return BsonValue.Null; } BsonArray asArray = value.AsArray; if (expr.Type == BsonExpressionType.Parameter) { BsonValue bsonValue = expr.ExecuteScalar(root, collation); if (!bsonValue.IsNumber) { throw new LiteException(0, "Parameter expression must return number when called inside an array"); } index = bsonValue.AsInt32; } int num = ((index < 0) ? (asArray.Count + index) : index); if (asArray.Count > num) { return asArray[num]; } return BsonValue.Null; } public static IEnumerable ARRAY_FILTER(BsonValue value, int index, BsonExpression filterExpr, BsonDocument root, Collation collation, BsonDocument parameters) { if (!value.IsArray) { yield break; } BsonArray asArray = value.AsArray; if (index == int.MaxValue) { foreach (BsonValue item in asArray) { yield return item; } yield break; } foreach (BsonValue item2 in asArray) { BsonValue bsonValue = filterExpr.ExecuteScalar(new BsonDocument[1] { root }, root, item2, collation); if (bsonValue.IsBoolean && bsonValue.AsBoolean) { yield return item2; } } } public static BsonValue DOCUMENT_INIT(string[] keys, BsonValue[] values) { Constants.ENSURE(keys.Length == values.Length, "both keys/value must contains same length"); if (keys.Length == 1 && keys[0][0] == '$' && values[0].IsString) { switch (keys[0]) { case "$binary": return BsonExpressionMethods.BINARY(values[0]); case "$oid": return BsonExpressionMethods.OBJECTID(values[0]); case "$guid": return BsonExpressionMethods.GUID(values[0]); case "$date": return BsonExpressionMethods.DATE(Collation.Binary, values[0]); case "$numberLong": return BsonExpressionMethods.LONG(values[0]); case "$numberDecimal": return BsonExpressionMethods.DECIMAL(Collation.Binary, values[0]); case "$minValue": return BsonExpressionMethods.MINVALUE(); case "$maxValue": return BsonExpressionMethods.MAXVALUE(); } } BsonDocument bsonDocument = new BsonDocument(); for (int i = 0; i < keys.Length; i++) { bsonDocument[keys[i]] = values[i]; } return bsonDocument; } public static BsonValue ARRAY_INIT(BsonValue[] values) { return new BsonArray(values); } } internal enum BsonExpressionParserMode { Full, Single, SelectDocument, UpdateDocument } internal class BsonExpressionParser { private static readonly Dictionary> _operators = new Dictionary> { ["%"] = Tuple.Create("%", M("MOD"), BsonExpressionType.Modulo), ["/"] = Tuple.Create("/", M("DIVIDE"), BsonExpressionType.Divide), ["*"] = Tuple.Create("*", M("MULTIPLY"), BsonExpressionType.Multiply), ["+"] = Tuple.Create("+", M("ADD"), BsonExpressionType.Add), ["-"] = Tuple.Create("-", M("MINUS"), BsonExpressionType.Subtract), ["LIKE"] = Tuple.Create(" LIKE ", M("LIKE"), BsonExpressionType.Like), ["BETWEEN"] = Tuple.Create(" BETWEEN ", M("BETWEEN"), BsonExpressionType.Between), ["IN"] = Tuple.Create(" IN ", M("IN"), BsonExpressionType.In), [">"] = Tuple.Create(">", M("GT"), BsonExpressionType.GreaterThan), [">="] = Tuple.Create(">=", M("GTE"), BsonExpressionType.GreaterThanOrEqual), ["<"] = Tuple.Create("<", M("LT"), BsonExpressionType.LessThan), ["<="] = Tuple.Create("<=", M("LTE"), BsonExpressionType.LessThanOrEqual), ["!="] = Tuple.Create("!=", M("NEQ"), BsonExpressionType.NotEqual), ["="] = Tuple.Create("=", M("EQ"), BsonExpressionType.Equal), ["ANY LIKE"] = Tuple.Create(" ANY LIKE ", M("LIKE_ANY"), BsonExpressionType.Like), ["ANY BETWEEN"] = Tuple.Create(" ANY BETWEEN ", M("BETWEEN_ANY"), BsonExpressionType.Between), ["ANY IN"] = Tuple.Create(" ANY IN ", M("IN_ANY"), BsonExpressionType.In), ["ANY >"] = Tuple.Create(" ANY>", M("GT_ANY"), BsonExpressionType.GreaterThan), ["ANY >="] = Tuple.Create(" ANY>=", M("GTE_ANY"), BsonExpressionType.GreaterThanOrEqual), ["ANY <"] = Tuple.Create(" ANY<", M("LT_ANY"), BsonExpressionType.LessThan), ["ANY <="] = Tuple.Create(" ANY<=", M("LTE_ANY"), BsonExpressionType.LessThanOrEqual), ["ANY !="] = Tuple.Create(" ANY!=", M("NEQ_ANY"), BsonExpressionType.NotEqual), ["ANY ="] = Tuple.Create(" ANY=", M("EQ_ANY"), BsonExpressionType.Equal), ["ALL LIKE"] = Tuple.Create(" ALL LIKE ", M("LIKE_ALL"), BsonExpressionType.Like), ["ALL BETWEEN"] = Tuple.Create(" ALL BETWEEN ", M("BETWEEN_ALL"), BsonExpressionType.Between), ["ALL IN"] = Tuple.Create(" ALL IN ", M("IN_ALL"), BsonExpressionType.In), ["ALL >"] = Tuple.Create(" ALL>", M("GT_ALL"), BsonExpressionType.GreaterThan), ["ALL >="] = Tuple.Create(" ALL>=", M("GTE_ALL"), BsonExpressionType.GreaterThanOrEqual), ["ALL <"] = Tuple.Create(" ALL<", M("LT_ALL"), BsonExpressionType.LessThan), ["ALL <="] = Tuple.Create(" ALL<=", M("LTE_ALL"), BsonExpressionType.LessThanOrEqual), ["ALL !="] = Tuple.Create(" ALL!=", M("NEQ_ALL"), BsonExpressionType.NotEqual), ["ALL ="] = Tuple.Create(" ALL=", M("EQ_ALL"), BsonExpressionType.Equal), ["AND"] = Tuple.Create(" AND ", null, BsonExpressionType.And), ["OR"] = Tuple.Create(" OR ", null, BsonExpressionType.Or) }; private static readonly MethodInfo _parameterPathMethod = M("PARAMETER_PATH"); private static readonly MethodInfo _memberPathMethod = M("MEMBER_PATH"); private static readonly MethodInfo _arrayIndexMethod = M("ARRAY_INDEX"); private static readonly MethodInfo _arrayFilterMethod = M("ARRAY_FILTER"); private static readonly MethodInfo _documentInitMethod = M("DOCUMENT_INIT"); private static readonly MethodInfo _arrayInitMethod = M("ARRAY_INIT"); private static readonly MethodInfo _itemsMethod = typeof(BsonExpressionMethods).GetMethod("ITEMS"); private static readonly MethodInfo _arrayMethod = typeof(BsonExpressionMethods).GetMethod("ARRAY"); private static MethodInfo M(string s) { return typeof(BsonExpressionOperators).GetMethod(s); } public static BsonExpression ParseFullExpression(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { BsonExpression item = ParseSingleExpression(tokenizer, context, parameters, scope); List list = new List { item }; List list2 = new List(); while (!tokenizer.EOF) { string text = ReadOperant(tokenizer); if (text == null) { break; } BsonExpression bsonExpression = ParseSingleExpression(tokenizer, context, parameters, scope); if (text.EndsWith("BETWEEN", StringComparison.OrdinalIgnoreCase)) { tokenizer.ReadToken().Expect("AND"); BsonExpression item2 = ParseSingleExpression(tokenizer, context, parameters, scope); bsonExpression = NewArray(bsonExpression, item2); } list.Add(bsonExpression); list2.Add(text.ToUpperInvariant()); } int num = 0; while (list.Count >= 2) { KeyValuePair> keyValuePair = _operators.ElementAt(num); int num2 = list2.IndexOf(keyValuePair.Key); if (num2 == -1) { num++; continue; } BsonExpression bsonExpression2 = list.ElementAt(num2); BsonExpression bsonExpression3 = list.ElementAt(num2 + 1); string item3 = keyValuePair.Value.Item1; MethodInfo item4 = keyValuePair.Value.Item2; BsonExpressionType item5 = keyValuePair.Value.Item3; int num3; if (!keyValuePair.Key.StartsWith("ALL")) { num3 = (keyValuePair.Key.StartsWith("ANY") ? 1 : 0); if (num3 == 0) { goto IL_0138; } } else { num3 = 1; } if (bsonExpression2.IsScalar) { bsonExpression2 = ConvertToEnumerable(bsonExpression2); } goto IL_0138; IL_0138: if (num3 == 0 && !bsonExpression2.IsScalar) { throw new LiteException(0, "Left expression `" + bsonExpression2.Source + "` returns more than one result. Try use ANY or ALL before operant."); } if (num3 == 0 && !bsonExpression3.IsScalar) { throw new LiteException(0, "Left expression `" + bsonExpression3.Source + "` must return a single value"); } if (!bsonExpression3.IsScalar) { throw new LiteException(0, "Right expression `" + bsonExpression3.Source + "` must return a single value"); } BsonExpression item6; if (item5 == BsonExpressionType.And || item5 == BsonExpressionType.Or) { item6 = CreateLogicExpression(item5, bsonExpression2, bsonExpression3); } else { List list3 = new List(); if (item4?.GetParameters().FirstOrDefault()?.ParameterType == typeof(Collation)) { list3.Add(context.Collation); } list3.Add(bsonExpression2.Expression); list3.Add(bsonExpression3.Expression); item6 = new BsonExpression { Type = item5, Parameters = parameters, IsImmutable = (bsonExpression2.IsImmutable && bsonExpression3.IsImmutable), UseSource = (bsonExpression2.UseSource || bsonExpression3.UseSource), IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(bsonExpression2.Fields).AddRange(bsonExpression3.Fields), Expression = Expression.Call(item4, list3.ToArray()), Left = bsonExpression2, Right = bsonExpression3, Source = bsonExpression2.Source + item3 + bsonExpression3.Source }; } list.Insert(num2, item6); list.RemoveRange(num2 + 1, 2); list2.RemoveAt(num2); } return list.Single(); } public static BsonExpression ParseSingleExpression(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { Token token = tokenizer.ReadToken(); return TryParseDouble(tokenizer, parameters) ?? TryParseInt(tokenizer, parameters) ?? TryParseBool(tokenizer, parameters) ?? TryParseNull(tokenizer, parameters) ?? TryParseString(tokenizer, parameters) ?? TryParseSource(tokenizer, context, parameters, scope) ?? TryParseDocument(tokenizer, context, parameters, scope) ?? TryParseArray(tokenizer, context, parameters, scope) ?? TryParseParameter(tokenizer, context, parameters, scope) ?? TryParseInnerExpression(tokenizer, context, parameters, scope) ?? TryParseFunction(tokenizer, context, parameters, scope) ?? TryParseMethodCall(tokenizer, context, parameters, scope) ?? TryParsePath(tokenizer, context, parameters, scope) ?? throw LiteException.UnexpectedToken(token); } public static BsonExpression ParseSelectDocumentBuilder(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters) { List> fields = new List>(); HashSet names = new HashSet(); int counter = 1; while (true) { BsonExpression bsonExpression = ParseFullExpression(tokenizer, context, parameters, DocumentScope.Root); Token token = tokenizer.LookAhead(); if (stop(token)) { Add(bsonExpression.DefaultFieldName(), bsonExpression); break; } if (token.Type == TokenType.Comma) { tokenizer.ReadToken(); Add(bsonExpression.DefaultFieldName(), bsonExpression); continue; } if (token.Is("AS")) { tokenizer.ReadToken(); } Add(tokenizer.ReadToken().Expect(TokenType.Word).Value, bsonExpression); token = tokenizer.LookAhead(); if (stop(token)) { break; } tokenizer.ReadToken().Expect(TokenType.Comma); } BsonExpression value = fields[0].Value; if (fields.Count == 1) { if (value.Type == BsonExpressionType.Path && value.Source == "$") { return BsonExpression.Root; } if (fields.Count == 1 && value.Type == BsonExpressionType.Document) { return value; } if (fields.Count == 1 && value.Type == BsonExpressionType.Call && value.Source.StartsWith("EXTEND")) { return value; } } Type typeFromHandle = typeof(string); Expression[] initializers = fields.Select((KeyValuePair x) => Expression.Constant(x.Key)).ToArray(); NewArrayExpression newArrayExpression = Expression.NewArrayInit(typeFromHandle, initializers); NewArrayExpression newArrayExpression2 = Expression.NewArrayInit(typeof(BsonValue), fields.Select((KeyValuePair x) => x.Value.Expression).ToArray()); BsonExpression bsonExpression2 = new BsonExpression(); bsonExpression2.Type = BsonExpressionType.Document; bsonExpression2.Parameters = parameters; bsonExpression2.IsImmutable = fields.All((KeyValuePair x) => x.Value.IsImmutable); bsonExpression2.UseSource = fields.Any((KeyValuePair x) => x.Value.UseSource); bsonExpression2.IsScalar = true; bsonExpression2.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(fields.SelectMany((KeyValuePair x) => x.Value.Fields)); bsonExpression2.Expression = Expression.Call(_documentInitMethod, new Expression[2] { newArrayExpression, newArrayExpression2 }); bsonExpression2.Source = "{" + string.Join(",", fields.Select((KeyValuePair x) => x.Key + ":" + x.Value.Source)) + "}"; return bsonExpression2; void Add(string alias, BsonExpression expr) { if (names.Contains(alias)) { alias += counter++; } names.Add(alias); if (!expr.IsScalar) { expr = ConvertToArray(expr); } fields.Add(new KeyValuePair(alias, expr)); } static bool stop(Token t) { if (!t.Is("FROM") && !t.Is("INTO") && t.Type != TokenType.EOF) { return t.Type == TokenType.SemiColon; } return true; } } public static BsonExpression ParseUpdateDocumentBuilder(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters) { if (tokenizer.LookAhead().Type == TokenType.OpenBrace) { tokenizer.ReadToken(); return TryParseDocument(tokenizer, context, parameters, DocumentScope.Root); } List list = new List(); List list2 = new List(); StringBuilder stringBuilder = new StringBuilder(); bool isImmutable = true; bool useSource = false; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); stringBuilder.Append("{"); while (!tokenizer.CheckEOF()) { string value = ReadKey(tokenizer, stringBuilder); tokenizer.ReadToken().Expect(TokenType.Equals); stringBuilder.Append(":"); BsonExpression bsonExpression = ParseFullExpression(tokenizer, context, parameters, DocumentScope.Root); if (!bsonExpression.IsScalar) { bsonExpression = ConvertToArray(bsonExpression); } if (!bsonExpression.IsImmutable) { isImmutable = false; } if (bsonExpression.UseSource) { useSource = true; } hashSet.AddRange(bsonExpression.Fields); list.Add(Expression.Constant(value)); list2.Add(bsonExpression.Expression); stringBuilder.Append(bsonExpression.Source); if (tokenizer.LookAhead().Type != TokenType.Comma) { break; } stringBuilder.Append(tokenizer.ReadToken().Value); } stringBuilder.Append("}"); NewArrayExpression newArrayExpression = Expression.NewArrayInit(typeof(string), list.ToArray()); NewArrayExpression newArrayExpression2 = Expression.NewArrayInit(typeof(BsonValue), list2.ToArray()); MethodCallExpression expression = Expression.Call(_documentInitMethod, new Expression[2] { newArrayExpression, newArrayExpression2 }); return new BsonExpression { Type = BsonExpressionType.Document, Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, IsScalar = true, Fields = hashSet, Expression = expression, Source = stringBuilder.ToString() }; } private static BsonExpression TryParseDouble(Tokenizer tokenizer, BsonDocument parameters) { string text = null; if (tokenizer.Current.Type == TokenType.Double) { text = tokenizer.Current.Value; } else if (tokenizer.Current.Type == TokenType.Minus && tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Double) { text = "-" + tokenizer.ReadToken().Value; } if (text != null) { double value = Convert.ToDouble(text, CultureInfo.InvariantCulture.NumberFormat); ConstantExpression expression = Expression.Constant(new BsonValue(value)); return new BsonExpression { Type = BsonExpressionType.Double, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression, Source = value.ToString("0.0########", CultureInfo.InvariantCulture.NumberFormat) }; } return null; } private static BsonExpression TryParseInt(Tokenizer tokenizer, BsonDocument parameters) { string text = null; if (tokenizer.Current.Type == TokenType.Int) { text = tokenizer.Current.Value; } else if (tokenizer.Current.Type == TokenType.Minus && tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Int) { text = "-" + tokenizer.ReadToken().Value; } if (text != null) { if (int.TryParse(text, NumberStyles.Any, CultureInfo.InvariantCulture.NumberFormat, out var result)) { ConstantExpression expression = Expression.Constant(new BsonValue(result)); return new BsonExpression { Type = BsonExpressionType.Int, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression, Source = result.ToString(CultureInfo.InvariantCulture.NumberFormat) }; } long value = long.Parse(text, NumberStyles.Any, CultureInfo.InvariantCulture.NumberFormat); ConstantExpression expression2 = Expression.Constant(new BsonValue(value)); return new BsonExpression { Type = BsonExpressionType.Int, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression2, Source = value.ToString(CultureInfo.InvariantCulture.NumberFormat) }; } return null; } private static BsonExpression TryParseBool(Tokenizer tokenizer, BsonDocument parameters) { if (tokenizer.Current.Type == TokenType.Word && (tokenizer.Current.Is("true") || tokenizer.Current.Is("false"))) { bool value = Convert.ToBoolean(tokenizer.Current.Value); ConstantExpression expression = Expression.Constant(new BsonValue(value)); return new BsonExpression { Type = BsonExpressionType.Boolean, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression, Source = value.ToString().ToLower() }; } return null; } private static BsonExpression TryParseNull(Tokenizer tokenizer, BsonDocument parameters) { if (tokenizer.Current.Type == TokenType.Word && tokenizer.Current.Is("null")) { ConstantExpression expression = Expression.Constant(BsonValue.Null); return new BsonExpression { Type = BsonExpressionType.Null, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression, Source = "null" }; } return null; } private static BsonExpression TryParseString(Tokenizer tokenizer, BsonDocument parameters) { if (tokenizer.Current.Type == TokenType.String) { BsonValue value = new BsonValue(tokenizer.Current.Value); ConstantExpression expression = Expression.Constant(value); return new BsonExpression { Type = BsonExpressionType.String, Parameters = parameters, IsImmutable = true, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = expression, Source = JsonSerializer.Serialize(value) }; } return null; } private static BsonExpression TryParseDocument(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != 0) { return null; } List list = new List(); List list2 = new List(); StringBuilder stringBuilder = new StringBuilder(); bool isImmutable = true; bool useSource = false; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); stringBuilder.Append("{"); BsonExpression bsonExpression2; if (tokenizer.LookAhead().Type == TokenType.CloseBrace) { stringBuilder.Append(tokenizer.ReadToken().Value); } else { while (!tokenizer.CheckEOF()) { StringBuilder stringBuilder2 = new StringBuilder(); string text = ReadKey(tokenizer, stringBuilder2); stringBuilder.Append((object?)stringBuilder2); tokenizer.ReadToken(); stringBuilder.Append(":"); BsonExpression bsonExpression; if (tokenizer.Current.Type == TokenType.Colon) { bsonExpression = ParseFullExpression(tokenizer, context, parameters, scope); tokenizer.ReadToken(); } else { string text2 = stringBuilder2.ToString(); bsonExpression2 = new BsonExpression(); bsonExpression2.Type = BsonExpressionType.Path; bsonExpression2.Parameters = parameters; bsonExpression2.IsImmutable = isImmutable; bsonExpression2.UseSource = useSource; bsonExpression2.IsScalar = true; bsonExpression2.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(new string[1] { text }); bsonExpression2.Expression = Expression.Call(_memberPathMethod, context.Root, Expression.Constant(text)); bsonExpression2.Source = "$." + (text2.IsWord() ? text2 : ("[" + text2 + "]")); bsonExpression = bsonExpression2; } if (!bsonExpression.IsScalar) { bsonExpression = ConvertToArray(bsonExpression); } if (!bsonExpression.IsImmutable) { isImmutable = false; } if (bsonExpression.UseSource) { useSource = true; } hashSet.AddRange(bsonExpression.Fields); list.Add(Expression.Constant(text)); list2.Add(bsonExpression.Expression); stringBuilder.Append(bsonExpression.Source); tokenizer.Current.Expect(TokenType.Comma, TokenType.CloseBrace); stringBuilder.Append(tokenizer.Current.Value); if (tokenizer.Current.Type != TokenType.Comma) { break; } } } NewArrayExpression newArrayExpression = Expression.NewArrayInit(typeof(string), list.ToArray()); NewArrayExpression newArrayExpression2 = Expression.NewArrayInit(typeof(BsonValue), list2.ToArray()); bsonExpression2 = new BsonExpression(); bsonExpression2.Type = BsonExpressionType.Document; bsonExpression2.Parameters = parameters; bsonExpression2.IsImmutable = isImmutable; bsonExpression2.UseSource = useSource; bsonExpression2.IsScalar = true; bsonExpression2.Fields = hashSet; bsonExpression2.Expression = Expression.Call(_documentInitMethod, new Expression[2] { newArrayExpression, newArrayExpression2 }); bsonExpression2.Source = stringBuilder.ToString(); return bsonExpression2; } private static BsonExpression TryParseSource(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.Asterisk) { return null; } BsonExpression bsonExpression = new BsonExpression { Type = BsonExpressionType.Source, Parameters = parameters, IsImmutable = true, UseSource = true, IsScalar = false, Fields = new HashSet(StringComparer.OrdinalIgnoreCase) { "$" }, Expression = context.Source, Source = "*" }; if (tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Period) { tokenizer.ReadToken(); BsonExpression bsonExpression2 = BsonExpression.ParseAndCompile(tokenizer, BsonExpressionParserMode.Single, parameters, DocumentScope.Source); if (bsonExpression2 == null) { throw LiteException.UnexpectedToken(tokenizer.Current); } return new BsonExpression { Type = BsonExpressionType.Map, Parameters = parameters, IsImmutable = bsonExpression2.IsImmutable, UseSource = true, IsScalar = false, Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(bsonExpression2.Fields), Expression = Expression.Call(BsonExpression.GetFunction("MAP"), context.Root, context.Collation, context.Parameters, bsonExpression.Expression, Expression.Constant(bsonExpression2)), Source = "MAP(*=>" + bsonExpression2.Source + ")" }; } return bsonExpression; } private static BsonExpression TryParseArray(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.OpenBracket) { return null; } List list = new List(); StringBuilder stringBuilder = new StringBuilder(); bool isImmutable = true; bool useSource = false; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); stringBuilder.Append("["); if (tokenizer.LookAhead().Type == TokenType.CloseBracket) { stringBuilder.Append(tokenizer.ReadToken().Value); } else { while (!tokenizer.CheckEOF()) { BsonExpression bsonExpression = ParseFullExpression(tokenizer, context, parameters, scope); if (!bsonExpression.IsScalar) { bsonExpression = ConvertToArray(bsonExpression); } stringBuilder.Append(bsonExpression.Source); if (!bsonExpression.IsImmutable) { isImmutable = false; } if (bsonExpression.UseSource) { useSource = true; } hashSet.AddRange(bsonExpression.Fields); list.Add(bsonExpression.Expression); Token token = tokenizer.ReadToken().Expect(TokenType.Comma, TokenType.CloseBracket); stringBuilder.Append(token.Value); if (token.Type != TokenType.Comma) { break; } } } NewArrayExpression arg = Expression.NewArrayInit(typeof(BsonValue), list.ToArray()); return new BsonExpression { Type = BsonExpressionType.Array, Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, IsScalar = true, Fields = hashSet, Expression = Expression.Call(_arrayInitMethod, arg), Source = stringBuilder.ToString() }; } private static BsonExpression TryParseParameter(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.At) { return null; } Token token = tokenizer.LookAhead(eatWhitespace: false); if (token.Type == TokenType.Word || token.Type == TokenType.Int) { string value = tokenizer.ReadToken(eatWhitespace: false).Value; ConstantExpression arg = Expression.Constant(value); return new BsonExpression { Type = BsonExpressionType.Parameter, Parameters = parameters, IsImmutable = false, UseSource = false, IsScalar = true, Fields = new HashSet(StringComparer.OrdinalIgnoreCase), Expression = Expression.Call(_parameterPathMethod, context.Parameters, arg), Source = "@" + value }; } return null; } private static BsonExpression TryParseInnerExpression(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.OpenParenthesis) { return null; } BsonExpression bsonExpression = ParseFullExpression(tokenizer, context, parameters, scope); tokenizer.ReadToken().Expect(TokenType.CloseParenthesis); return new BsonExpression { Type = bsonExpression.Type, Parameters = bsonExpression.Parameters, IsImmutable = bsonExpression.IsImmutable, UseSource = bsonExpression.UseSource, IsScalar = bsonExpression.IsScalar, Fields = bsonExpression.Fields, Expression = bsonExpression.Expression, Left = bsonExpression.Left, Right = bsonExpression.Right, Source = "(" + bsonExpression.Source + ")" }; } private static BsonExpression TryParseMethodCall(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { Token current = tokenizer.Current; if (tokenizer.Current.Type != TokenType.Word) { return null; } if (tokenizer.LookAhead().Type != TokenType.OpenParenthesis) { return null; } tokenizer.ReadToken(); List list = new List(); StringBuilder stringBuilder = new StringBuilder(); bool isImmutable = true; bool useSource = false; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); stringBuilder.Append(current.Value.ToUpperInvariant() + "("); if (tokenizer.LookAhead().Type == TokenType.CloseParenthesis) { stringBuilder.Append(tokenizer.ReadToken().Value); } else { while (!tokenizer.CheckEOF()) { BsonExpression bsonExpression = ParseFullExpression(tokenizer, context, parameters, scope); if (!bsonExpression.IsImmutable) { isImmutable = false; } if (bsonExpression.UseSource) { useSource = true; } hashSet.AddRange(bsonExpression.Fields); list.Add(bsonExpression); stringBuilder.Append(bsonExpression.Source); Token token = tokenizer.ReadToken().Expect(TokenType.Comma, TokenType.CloseParenthesis); stringBuilder.Append(token.Value); if (token.Type != TokenType.Comma) { break; } } } MethodInfo method = BsonExpression.GetMethod(current.Value, list.Count); if (method == null) { throw LiteException.UnexpectedToken("Method '" + current.Value.ToUpperInvariant() + "' does not exist or contains invalid parameters", current); } if (method.GetCustomAttribute() != null) { isImmutable = false; } List list2 = new List(); if (method.GetParameters().FirstOrDefault()?.ParameterType == typeof(Collation)) { list2.Add(context.Collation); } foreach (var item in (from x in method.GetParameters() where x.ParameterType != typeof(Collation) select x).Zip(list, (ParameterInfo parameter, BsonExpression expr) => new { parameter, expr })) { if (!item.parameter.ParameterType.IsEnumerable() && !item.expr.IsScalar) { list2.Add(ConvertToArray(item.expr).Expression); } else if (item.parameter.ParameterType.IsEnumerable() && item.expr.IsScalar) { list2.Add(ConvertToEnumerable(item.expr).Expression); } else { list2.Add(item.expr.Expression); } } if (method.Name == "IIF" && list.Count == 3) { return CreateConditionalExpression(list[0], list[1], list[2]); } return new BsonExpression { Type = BsonExpressionType.Call, Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, IsScalar = !method.ReturnType.IsEnumerable(), Fields = hashSet, Expression = Expression.Call(method, list2.ToArray()), Source = stringBuilder.ToString() }; } private static BsonExpression TryParsePath(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.At && tokenizer.Current.Type != TokenType.Dollar && tokenizer.Current.Type != TokenType.Word) { return null; } TokenType tokenType = ((scope == DocumentScope.Root) ? TokenType.Dollar : TokenType.At); if (tokenizer.Current.Type == TokenType.At || tokenizer.Current.Type == TokenType.Dollar) { tokenType = tokenizer.Current.Type; if (tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Period) { tokenizer.ReadToken(); tokenizer.ReadToken(); } } StringBuilder stringBuilder = new StringBuilder(); bool isImmutable = true; bool useSource = false; bool isScalar = true; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); stringBuilder.Append((tokenType == TokenType.Dollar) ? "$" : "@"); string text = ReadField(tokenizer, stringBuilder); ConstantExpression arg = Expression.Constant(text); Expression expression = Expression.Call(_memberPathMethod, (tokenType == TokenType.Dollar) ? context.Root : context.Current, arg); if (tokenType == TokenType.Dollar || scope == DocumentScope.Source) { hashSet.Add((text.Length == 0) ? "$" : text); } while (!tokenizer.EOF) { Expression expression2 = ParsePath(tokenizer, expression, context, parameters, hashSet, ref isImmutable, ref useSource, ref isScalar, stringBuilder); if (!isScalar) { expression = expression2; break; } if (expression2 == null) { break; } expression = expression2; } BsonExpression bsonExpression = new BsonExpression { Type = BsonExpressionType.Path, Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, IsScalar = isScalar, Fields = hashSet, Expression = expression, Source = stringBuilder.ToString() }; if (!isScalar && tokenizer.LookAhead(eatWhitespace: false).Type == TokenType.Period) { tokenizer.ReadToken(); BsonExpression bsonExpression2 = BsonExpression.ParseAndCompile(tokenizer, BsonExpressionParserMode.Single, parameters, DocumentScope.Current); if (bsonExpression2 == null) { throw LiteException.UnexpectedToken(tokenizer.Current); } BsonExpression bsonExpression3 = new BsonExpression(); bsonExpression3.Type = BsonExpressionType.Map; bsonExpression3.Parameters = parameters; bsonExpression3.IsImmutable = bsonExpression.IsImmutable && bsonExpression2.IsImmutable; bsonExpression3.UseSource = bsonExpression.UseSource || bsonExpression2.UseSource; bsonExpression3.IsScalar = false; bsonExpression3.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(bsonExpression.Fields).AddRange(bsonExpression2.Fields); bsonExpression3.Expression = Expression.Call(BsonExpression.GetFunction("MAP"), context.Root, context.Collation, context.Parameters, bsonExpression.Expression, Expression.Constant(bsonExpression2)); bsonExpression3.Source = "MAP(" + bsonExpression.Source + "=>" + bsonExpression2.Source + ")"; return bsonExpression3; } return bsonExpression; } private static Expression ParsePath(Tokenizer tokenizer, Expression expr, ExpressionContext context, BsonDocument parameters, HashSet fields, ref bool isImmutable, ref bool useSource, ref bool isScalar, StringBuilder src) { Token token = tokenizer.LookAhead(eatWhitespace: false); if (token.Type == TokenType.Period) { tokenizer.ReadToken(); tokenizer.ReadToken(eatWhitespace: false); ConstantExpression arg = Expression.Constant(ReadField(tokenizer, src)); return Expression.Call(_memberPathMethod, expr, arg); } if (token.Type == TokenType.OpenBracket) { src.Append("["); tokenizer.ReadToken(); token = tokenizer.LookAhead(); int num = 0; BsonExpression bsonExpression = new BsonExpression(); MethodInfo method = _arrayIndexMethod; if (token.Type == TokenType.Int) { src.Append(tokenizer.ReadToken().Value); num = Convert.ToInt32(tokenizer.Current.Value); } else if (token.Type == TokenType.Minus) { src.Append(tokenizer.ReadToken().Value + tokenizer.ReadToken().Expect(TokenType.Int).Value); num = -Convert.ToInt32(tokenizer.Current.Value); } else if (token.Type == TokenType.Asterisk) { method = _arrayFilterMethod; isScalar = false; num = int.MaxValue; src.Append(tokenizer.ReadToken().Value); } else { bsonExpression = BsonExpression.ParseAndCompile(tokenizer, BsonExpressionParserMode.Full, parameters, DocumentScope.Current); if (bsonExpression == null) { throw LiteException.UnexpectedToken(tokenizer.Current); } if (!bsonExpression.IsImmutable) { isImmutable = false; } if (bsonExpression.UseSource) { useSource = true; } if (bsonExpression.Type != BsonExpressionType.Parameter) { method = _arrayFilterMethod; isScalar = false; } fields.AddRange(bsonExpression.Fields); src.Append(bsonExpression.Source); } tokenizer.ReadToken().Expect(TokenType.CloseBracket); src.Append("]"); return Expression.Call(method, expr, Expression.Constant(num), Expression.Constant(bsonExpression), context.Root, context.Collation, context.Parameters); } return null; } private static BsonExpression TryParseFunction(Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.Current.Type != TokenType.Word) { return null; } if (tokenizer.LookAhead().Type != TokenType.OpenParenthesis) { return null; } string text = tokenizer.Current.Value.ToUpperInvariant(); return text switch { "MAP" => ParseFunction(text, BsonExpressionType.Map, tokenizer, context, parameters, scope), "FILTER" => ParseFunction(text, BsonExpressionType.Filter, tokenizer, context, parameters, scope), "SORT" => ParseFunction(text, BsonExpressionType.Sort, tokenizer, context, parameters, scope), _ => null, }; } private static BsonExpression ParseFunction(string functionName, BsonExpressionType type, Tokenizer tokenizer, ExpressionContext context, BsonDocument parameters, DocumentScope scope) { if (tokenizer.LookAhead().Type != TokenType.OpenParenthesis) { return null; } tokenizer.ReadToken().Expect(TokenType.OpenParenthesis); BsonExpression bsonExpression = ParseSingleExpression(tokenizer, context, parameters, scope); if (bsonExpression.IsScalar) { bsonExpression = ConvertToEnumerable(bsonExpression); } List list = new List(); list.Add(context.Root); list.Add(context.Collation); list.Add(context.Parameters); StringBuilder stringBuilder = new StringBuilder(functionName + "(" + bsonExpression.Source); bool isImmutable = bsonExpression.IsImmutable; bool useSource = bsonExpression.UseSource; HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); list.Add(bsonExpression.Expression); hashSet.AddRange(bsonExpression.Fields); if (tokenizer.LookAhead().Type == TokenType.Equals) { tokenizer.ReadToken().Expect(TokenType.Equals); tokenizer.ReadToken().Expect(TokenType.Greater); BsonExpression bsonExpression2 = BsonExpression.ParseAndCompile(tokenizer, BsonExpressionParserMode.Full, parameters, (bsonExpression.Type != BsonExpressionType.Source) ? DocumentScope.Current : DocumentScope.Source); stringBuilder.Append("=>" + bsonExpression2.Source); list.Add(Expression.Constant(bsonExpression2)); hashSet.AddRange(bsonExpression2.Fields); } if (tokenizer.LookAhead().Type != TokenType.CloseParenthesis) { tokenizer.ReadToken().Expect(TokenType.Comma); stringBuilder.Append(","); while (!tokenizer.CheckEOF()) { BsonExpression bsonExpression3 = ParseFullExpression(tokenizer, context, parameters, scope); if (!bsonExpression3.IsImmutable) { isImmutable = false; } if (bsonExpression3.UseSource) { useSource = true; } list.Add(bsonExpression3.Expression); stringBuilder.Append(bsonExpression3.Source); hashSet.AddRange(bsonExpression3.Fields); if (tokenizer.LookAhead().Type != TokenType.Comma) { break; } stringBuilder.Append(tokenizer.ReadToken().Value); } } tokenizer.ReadToken().Expect(TokenType.CloseParenthesis); stringBuilder.Append(")"); MethodInfo function = BsonExpression.GetFunction(functionName, list.Count - 5); return new BsonExpression { Type = type, Parameters = parameters, IsImmutable = isImmutable, UseSource = useSource, IsScalar = false, Fields = hashSet, Expression = Expression.Call(function, list.ToArray()), Source = stringBuilder.ToString() }; } private static BsonExpression NewArray(BsonExpression item0, BsonExpression item1) { Expression[] source = new Expression[2] { item0.Expression, item1.Expression }; if (!item0.IsScalar) { throw new LiteException(0, "Expression `" + item0.Source + "` must be a scalar expression"); } if (!item1.IsScalar) { throw new LiteException(0, "Expression `" + item0.Source + "` must be a scalar expression"); } NewArrayExpression newArrayExpression = Expression.NewArrayInit(typeof(BsonValue), source.ToArray()); BsonExpression bsonExpression = new BsonExpression(); bsonExpression.Type = BsonExpressionType.Array; bsonExpression.Parameters = item0.Parameters; bsonExpression.IsImmutable = item0.IsImmutable && item1.IsImmutable; bsonExpression.UseSource = item0.UseSource || item1.UseSource; bsonExpression.IsScalar = true; bsonExpression.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(item0.Fields).AddRange(item1.Fields); bsonExpression.Expression = Expression.Call(_arrayInitMethod, new Expression[1] { newArrayExpression }); bsonExpression.Source = item0.Source + " AND " + item1.Source; return bsonExpression; } private static string ReadField(Tokenizer tokenizer, StringBuilder source) { string text = ""; if (tokenizer.Current.Type == TokenType.OpenBracket) { text = tokenizer.ReadToken().Expect(TokenType.String).Value; tokenizer.ReadToken().Expect(TokenType.CloseBracket); } else if (tokenizer.Current.Type == TokenType.Word) { text = tokenizer.Current.Value; } if (text.Length > 0) { source.Append("."); if (text.IsWord()) { source.Append(text); } else { source.Append("["); JsonSerializer.Serialize(text, source); source.Append("]"); } } return text; } public static string ReadKey(Tokenizer tokenizer, StringBuilder source) { Token token = tokenizer.ReadToken(); string text = ""; text = ((token.Type != TokenType.String) ? token.Expect(TokenType.Word, TokenType.Int).Value : token.Value); if (text.IsWord()) { source.Append(text); } else { JsonSerializer.Serialize(text, source); } return text; } private static string ReadOperant(Tokenizer tokenizer) { Token token = tokenizer.LookAhead(); if (token.IsOperand) { tokenizer.ReadToken(); return token.Value; } if (token.Is("ALL") || token.Is("ANY")) { string text = token.Value.ToUpperInvariant(); tokenizer.ReadToken(); token = tokenizer.ReadToken(); if (!token.IsOperand) { throw LiteException.UnexpectedToken("Expected valid operand", token); } return text + " " + token.Value; } return null; } private static BsonExpression ConvertToEnumerable(BsonExpression expr) { string source = ((expr.Type == BsonExpressionType.Path) ? (expr.Source + "[*]") : ("ITEMS(" + expr.Source + ")")); BsonExpressionType type = ((expr.Type == BsonExpressionType.Path) ? BsonExpressionType.Path : BsonExpressionType.Call); return new BsonExpression { Type = type, Parameters = expr.Parameters, IsImmutable = expr.IsImmutable, UseSource = expr.UseSource, IsScalar = false, Fields = expr.Fields, Expression = Expression.Call(_itemsMethod, expr.Expression), Source = source }; } private static BsonExpression ConvertToArray(BsonExpression expr) { return new BsonExpression { Type = BsonExpressionType.Call, Parameters = expr.Parameters, IsImmutable = expr.IsImmutable, UseSource = expr.UseSource, IsScalar = true, Fields = expr.Fields, Expression = Expression.Call(_arrayMethod, expr.Expression), Source = "ARRAY(" + expr.Source + ")" }; } internal static BsonExpression CreateLogicExpression(BsonExpressionType type, BsonExpression left, BsonExpression right) { MemberExpression left2 = Expression.Property(left.Expression, typeof(BsonValue), "AsBoolean"); MemberExpression right2 = Expression.Property(right.Expression, typeof(BsonValue), "AsBoolean"); BinaryExpression binaryExpression = ((type == BsonExpressionType.And) ? Expression.AndAlso(left2, right2) : Expression.OrElse(left2, right2)); ConstructorInfo constructor = typeof(BsonValue).GetConstructors().First((ConstructorInfo x) => x.GetParameters().FirstOrDefault()?.ParameterType == typeof(bool)); BsonExpression bsonExpression = new BsonExpression(); bsonExpression.Type = type; bsonExpression.Parameters = left.Parameters; bsonExpression.IsImmutable = left.IsImmutable && right.IsImmutable; bsonExpression.UseSource = left.UseSource || right.UseSource; bsonExpression.IsScalar = left.IsScalar && right.IsScalar; bsonExpression.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(left.Fields).AddRange(right.Fields); bsonExpression.Expression = Expression.New(constructor, binaryExpression); bsonExpression.Left = left; bsonExpression.Right = right; bsonExpression.Source = left.Source + " " + type.ToString().ToUpperInvariant() + " " + right.Source; return bsonExpression; } internal static BsonExpression CreateConditionalExpression(BsonExpression test, BsonExpression ifTrue, BsonExpression ifFalse) { ConditionalExpression expression = Expression.Condition(Expression.Property(test.Expression, typeof(BsonValue), "AsBoolean"), ifTrue.Expression, ifFalse.Expression); BsonExpression bsonExpression = new BsonExpression(); bsonExpression.Type = BsonExpressionType.Call; bsonExpression.Parameters = test.Parameters; bsonExpression.IsImmutable = (test.IsImmutable && ifTrue.IsImmutable) || ifFalse.IsImmutable; bsonExpression.UseSource = test.UseSource || ifTrue.UseSource || ifFalse.UseSource; bsonExpression.IsScalar = test.IsScalar && ifTrue.IsScalar && ifFalse.IsScalar; bsonExpression.Fields = new HashSet(StringComparer.OrdinalIgnoreCase).AddRange(test.Fields).AddRange(ifTrue.Fields).AddRange(ifFalse.Fields); bsonExpression.Expression = expression; bsonExpression.Source = "IIF(" + test.Source + "," + ifTrue.Source + "," + ifFalse.Source + ")"; return bsonExpression; } } public enum BsonExpressionType : byte { Double = 1, Int, String, Boolean, Null, Array, Document, Parameter, Call, Path, Modulo, Add, Subtract, Multiply, Divide, Equal, Like, Between, GreaterThan, GreaterThanOrEqual, LessThan, LessThanOrEqual, NotEqual, In, Or, And, Map, Filter, Sort, Source } internal enum DocumentScope { Source, Root, Current } internal class ExpressionContext { public ParameterExpression Source { get; } public ParameterExpression Root { get; } public ParameterExpression Current { get; } public ParameterExpression Collation { get; } public ParameterExpression Parameters { get; } public ExpressionContext() { Source = Expression.Parameter(typeof(IEnumerable), "source"); Root = Expression.Parameter(typeof(BsonDocument), "root"); Current = Expression.Parameter(typeof(BsonValue), "current"); Collation = Expression.Parameter(typeof(Collation), "collation"); Parameters = Expression.Parameter(typeof(BsonDocument), "parameters"); } } public class JsonReader { private static readonly IFormatProvider _numberFormat = CultureInfo.InvariantCulture.NumberFormat; private readonly Tokenizer _tokenizer; public long Position => _tokenizer.Position; public JsonReader(TextReader reader) { if (reader == null) { throw new ArgumentNullException("reader"); } _tokenizer = new Tokenizer(reader); } internal JsonReader(Tokenizer tokenizer) { _tokenizer = tokenizer ?? throw new ArgumentNullException("tokenizer"); } public BsonValue Deserialize() { Token token = _tokenizer.ReadToken(); if (token.Type == TokenType.EOF) { return BsonValue.Null; } return ReadValue(token); } public IEnumerable DeserializeArray() { Token token = _tokenizer.ReadToken(); if (token.Type == TokenType.EOF) { yield break; } token.Expect(TokenType.OpenBracket); token = _tokenizer.ReadToken(); while (token.Type != TokenType.CloseBracket) { yield return ReadValue(token); token = _tokenizer.ReadToken(); if (token.Type == TokenType.Comma) { token = _tokenizer.ReadToken(); } } token.Expect(TokenType.CloseBracket); } internal BsonValue ReadValue(Token token) { string text = token.Value; switch (token.Type) { case TokenType.String: return text; case TokenType.OpenBrace: return ReadObject(); case TokenType.OpenBracket: return ReadArray(); case TokenType.Minus: { Token token2 = _tokenizer.ReadToken(eatWhitespace: false).Expect(TokenType.Int, TokenType.Double); text = "-" + token2.Value; if (token2.Type == TokenType.Int) { goto case TokenType.Int; } if (token2.Type != TokenType.Double) { break; } goto case TokenType.Double; } case TokenType.Int: { if (int.TryParse(text, NumberStyles.Any, _numberFormat, out var result)) { return new BsonValue(result); } return new BsonValue(long.Parse(text, NumberStyles.Any, _numberFormat)); } case TokenType.Double: return new BsonValue(Convert.ToDouble(text, _numberFormat)); case TokenType.Word: return text.ToLower() switch { "null" => BsonValue.Null, "true" => true, "false" => false, _ => throw LiteException.UnexpectedToken(token), }; } throw LiteException.UnexpectedToken(token); } private BsonValue ReadObject() { BsonDocument bsonDocument = new BsonDocument(); Token token = _tokenizer.ReadToken(); while (token.Type != TokenType.CloseBrace) { token.Expect(TokenType.String, TokenType.Word); string value = token.Value; token = _tokenizer.ReadToken(); token.Expect(TokenType.Colon); token = _tokenizer.ReadToken(); if (value[0] == '$' && bsonDocument.Count == 0) { BsonValue bsonValue = ReadExtendedDataType(value, token.Value); if (!bsonValue.IsNull) { return bsonValue; } } bsonDocument[value] = ReadValue(token); token = _tokenizer.ReadToken(); if (token.Type == TokenType.Comma) { token = _tokenizer.ReadToken(); } } return bsonDocument; } private BsonArray ReadArray() { BsonArray bsonArray = new BsonArray(); Token token = _tokenizer.ReadToken(); while (token.Type != TokenType.CloseBracket) { BsonValue item = ReadValue(token); bsonArray.Add(item); token = _tokenizer.ReadToken(); if (token.Type == TokenType.Comma) { token = _tokenizer.ReadToken(); } } return bsonArray; } private BsonValue ReadExtendedDataType(string key, string value) { BsonValue result; switch (key) { case "$binary": result = new BsonValue(Convert.FromBase64String(value)); break; case "$oid": result = new BsonValue(new ObjectId(value)); break; case "$guid": result = new BsonValue(new Guid(value)); break; case "$date": result = new BsonValue(DateTime.Parse(value).ToLocalTime()); break; case "$numberLong": result = new BsonValue(Convert.ToInt64(value, _numberFormat)); break; case "$numberDecimal": result = new BsonValue(Convert.ToDecimal(value, _numberFormat)); break; case "$minValue": result = BsonValue.MinValue; break; case "$maxValue": result = BsonValue.MaxValue; break; default: return BsonValue.Null; } _tokenizer.ReadToken().Expect(TokenType.CloseBrace); return result; } } public class JsonSerializer { public static string Serialize(BsonValue value, bool indent = false) { StringBuilder stringBuilder = new StringBuilder(); Serialize(value, stringBuilder, indent); return stringBuilder.ToString(); } public static void Serialize(BsonValue value, TextWriter writer, bool indent = false) { JsonWriter jsonWriter = new JsonWriter(writer); jsonWriter.Pretty = indent; jsonWriter.Serialize(value ?? BsonValue.Null); } public static void Serialize(BsonValue value, StringBuilder sb, bool indent = false) { using StringWriter writer = new StringWriter(sb); JsonWriter jsonWriter = new JsonWriter(writer); jsonWriter.Pretty = indent; jsonWriter.Serialize(value ?? BsonValue.Null); } public static BsonValue Deserialize(string json) { if (json == null) { throw new ArgumentNullException("json"); } using StringReader reader = new StringReader(json); return new JsonReader(reader).Deserialize(); } public static BsonValue Deserialize(TextReader reader) { if (reader == null) { throw new ArgumentNullException("reader"); } return new JsonReader(reader).Deserialize(); } public static IEnumerable DeserializeArray(string json) { if (json == null) { throw new ArgumentNullException("json"); } return new JsonReader(new StringReader(json)).DeserializeArray(); } public static IEnumerable DeserializeArray(TextReader reader) { if (reader == null) { throw new ArgumentNullException("reader"); } return new JsonReader(reader).DeserializeArray(); } } public class JsonWriter { private static readonly IFormatProvider _numberFormat = CultureInfo.InvariantCulture.NumberFormat; private readonly TextWriter _writer; private int _indent; private string _spacer = ""; public int Indent { get; set; } = 4; public bool Pretty { get; set; } public JsonWriter(TextWriter writer) { _writer = writer; } public void Serialize(BsonValue value) { _indent = 0; _spacer = (Pretty ? " " : ""); WriteValue(value ?? BsonValue.Null); } private void WriteValue(BsonValue value) { switch (value.Type) { case BsonType.Null: _writer.Write("null"); break; case BsonType.Array: WriteArray(value.AsArray); break; case BsonType.Document: WriteObject(value.AsDocument); break; case BsonType.Boolean: _writer.Write(value.AsBoolean.ToString().ToLower()); break; case BsonType.String: WriteString(value.AsString); break; case BsonType.Int32: _writer.Write(value.AsInt32.ToString(_numberFormat)); break; case BsonType.Double: { double asDouble = value.AsDouble; if (double.IsNaN(asDouble) || double.IsNegativeInfinity(asDouble) || double.IsPositiveInfinity(asDouble)) { _writer.Write("null"); } else { _writer.Write(value.AsDouble.ToString("0.0########", _numberFormat)); } break; } case BsonType.Binary: { byte[] asBinary = value.AsBinary; WriteExtendDataType("$binary", Convert.ToBase64String(asBinary, 0, asBinary.Length)); break; } case BsonType.ObjectId: WriteExtendDataType("$oid", value.AsObjectId.ToString()); break; case BsonType.Guid: WriteExtendDataType("$guid", value.AsGuid.ToString()); break; case BsonType.DateTime: WriteExtendDataType("$date", value.AsDateTime.ToUniversalTime().ToString("o")); break; case BsonType.Int64: WriteExtendDataType("$numberLong", value.AsInt64.ToString(_numberFormat)); break; case BsonType.Decimal: WriteExtendDataType("$numberDecimal", value.AsDecimal.ToString(_numberFormat)); break; case BsonType.MinValue: WriteExtendDataType("$minValue", "1"); break; case BsonType.MaxValue: WriteExtendDataType("$maxValue", "1"); break; } } private void WriteObject(BsonDocument obj) { int num = obj.Keys.Count(); bool hasData = num > 0; WriteStartBlock("{", hasData); int num2 = 0; foreach (KeyValuePair element in obj.GetElements()) { WriteKeyValue(element.Key, element.Value, num2++ < num - 1); } WriteEndBlock("}", hasData); } private void WriteArray(BsonArray arr) { bool hasData = arr.Count > 0; WriteStartBlock("[", hasData); for (int i = 0; i < arr.Count; i++) { BsonValue bsonValue = arr[i]; if (Pretty && bsonValue != null && (!bsonValue.IsDocument || !bsonValue.AsDocument.Keys.Any()) && (!bsonValue.IsArray || bsonValue.AsArray.Count <= 0)) { WriteIndent(); } WriteValue(bsonValue ?? BsonValue.Null); if (i < arr.Count - 1) { _writer.Write(','); } WriteNewLine(); } WriteEndBlock("]", hasData); } private void WriteString(string s) { _writer.Write('"'); int length = s.Length; for (int i = 0; i < length; i++) { char c = s[i]; switch (c) { case '"': _writer.Write("\\\""); continue; case '\\': _writer.Write("\\\\"); continue; case '\b': _writer.Write("\\b"); continue; case '\f': _writer.Write("\\f"); continue; case '\n': _writer.Write("\\n"); continue; case '\r': _writer.Write("\\r"); continue; case '\t': _writer.Write("\\t"); continue; } switch (CharUnicodeInfo.GetUnicodeCategory(c)) { case UnicodeCategory.UppercaseLetter: case UnicodeCategory.LowercaseLetter: case UnicodeCategory.TitlecaseLetter: case UnicodeCategory.OtherLetter: case UnicodeCategory.DecimalDigitNumber: case UnicodeCategory.LetterNumber: case UnicodeCategory.OtherNumber: case UnicodeCategory.SpaceSeparator: case UnicodeCategory.ConnectorPunctuation: case UnicodeCategory.DashPunctuation: case UnicodeCategory.OpenPunctuation: case UnicodeCategory.ClosePunctuation: case UnicodeCategory.InitialQuotePunctuation: case UnicodeCategory.FinalQuotePunctuation: case UnicodeCategory.OtherPunctuation: case UnicodeCategory.MathSymbol: case UnicodeCategory.CurrencySymbol: case UnicodeCategory.ModifierSymbol: case UnicodeCategory.OtherSymbol: _writer.Write(c); continue; } _writer.Write("\\u"); TextWriter writer = _writer; int num = c; writer.Write(num.ToString("x04")); } _writer.Write('"'); } private void WriteExtendDataType(string type, string value) { _writer.Write("{\""); _writer.Write(type); _writer.Write("\":"); _writer.Write(_spacer); _writer.Write("\""); _writer.Write(value); _writer.Write("\"}"); } private void WriteKeyValue(string key, BsonValue value, bool comma) { WriteIndent(); _writer.Write('"'); _writer.Write(key); _writer.Write("\":"); if (Pretty) { _writer.Write(' '); if (value != null && ((value.IsDocument && value.AsDocument.Keys.Any()) || (value.IsArray && value.AsArray.Count > 0))) { WriteNewLine(); } } WriteValue(value ?? BsonValue.Null); if (comma) { _writer.Write(','); } WriteNewLine(); } private void WriteStartBlock(string str, bool hasData) { if (hasData) { WriteIndent(); _writer.Write(str); WriteNewLine(); _indent++; } else { _writer.Write(str); } } private void WriteEndBlock(string str, bool hasData) { if (hasData) { _indent--; WriteIndent(); _writer.Write(str); } else { _writer.Write(str); } } private void WriteNewLine() { if (Pretty) { _writer.WriteLine(); } } private void WriteIndent() { if (Pretty) { _writer.Write("".PadRight(_indent * Indent, ' ')); } } } public class ObjectId : IComparable, IEquatable { private static readonly int _machine; private static readonly short _pid; private static int _increment; public static ObjectId Empty => new ObjectId(); public int Timestamp { get; } public int Machine { get; } public short Pid { get; } public int Increment { get; } public DateTime CreationTime { get { DateTime unixEpoch = BsonValue.UnixEpoch; return unixEpoch.AddSeconds(Timestamp); } } public ObjectId() { Timestamp = 0; Machine = 0; Pid = 0; Increment = 0; } public ObjectId(int timestamp, int machine, short pid, int increment) { Timestamp = timestamp; Machine = machine; Pid = pid; Increment = increment; } public ObjectId(ObjectId from) { Timestamp = from.Timestamp; Machine = from.Machine; Pid = from.Pid; Increment = from.Increment; } public ObjectId(string value) : this(FromHex(value)) { } public ObjectId(byte[] bytes, int startIndex = 0) { if (bytes == null) { throw new ArgumentNullException("bytes"); } Timestamp = (bytes[startIndex] << 24) + (bytes[startIndex + 1] << 16) + (bytes[startIndex + 2] << 8) + bytes[startIndex + 3]; Machine = (bytes[startIndex + 4] << 16) + (bytes[startIndex + 5] << 8) + bytes[startIndex + 6]; Pid = (short)((bytes[startIndex + 7] << 8) + bytes[startIndex + 8]); Increment = (bytes[startIndex + 9] << 16) + (bytes[startIndex + 10] << 8) + bytes[startIndex + 11]; } private static byte[] FromHex(string value) { if (string.IsNullOrEmpty(value)) { throw new ArgumentNullException("value"); } if (value.Length != 24) { throw new ArgumentException($"ObjectId strings should be 24 hex characters, got {value.Length} : \"{value}\""); } byte[] array = new byte[12]; for (int i = 0; i < 24; i += 2) { array[i / 2] = Convert.ToByte(value.Substring(i, 2), 16); } return array; } public bool Equals(ObjectId other) { if (other != null && Timestamp == other.Timestamp && Machine == other.Machine && Pid == other.Pid) { return Increment == other.Increment; } return false; } public override bool Equals(object other) { return Equals(other as ObjectId); } public override int GetHashCode() { int num = 17; num = 37 * num + Timestamp.GetHashCode(); num = 37 * num + Machine.GetHashCode(); num = 37 * num + Pid.GetHashCode(); return 37 * num + Increment.GetHashCode(); } public int CompareTo(ObjectId other) { int num = Timestamp.CompareTo(other.Timestamp); if (num != 0) { return num; } num = Machine.CompareTo(other.Machine); if (num != 0) { return num; } num = Pid.CompareTo(other.Pid); if (num != 0) { if (num >= 0) { return 1; } return -1; } return Increment.CompareTo(other.Increment); } public void ToByteArray(byte[] bytes, int startIndex) { bytes[startIndex] = (byte)(Timestamp >> 24); bytes[startIndex + 1] = (byte)(Timestamp >> 16); bytes[startIndex + 2] = (byte)(Timestamp >> 8); bytes[startIndex + 3] = (byte)Timestamp; bytes[startIndex + 4] = (byte)(Machine >> 16); bytes[startIndex + 5] = (byte)(Machine >> 8); bytes[startIndex + 6] = (byte)Machine; bytes[startIndex + 7] = (byte)(Pid >> 8); bytes[startIndex + 8] = (byte)Pid; bytes[startIndex + 9] = (byte)(Increment >> 16); bytes[startIndex + 10] = (byte)(Increment >> 8); bytes[startIndex + 11] = (byte)Increment; } public byte[] ToByteArray() { byte[] array = new byte[12]; ToByteArray(array, 0); return array; } public override string ToString() { return BitConverter.ToString(ToByteArray()).Replace("-", "").ToLower(); } public static bool operator ==(ObjectId lhs, ObjectId rhs) { if ((object)lhs == null) { return (object)rhs == null; } if ((object)rhs == null) { return false; } return lhs.Equals(rhs); } public static bool operator !=(ObjectId lhs, ObjectId rhs) { return !(lhs == rhs); } public static bool operator >=(ObjectId lhs, ObjectId rhs) { return lhs.CompareTo(rhs) >= 0; } public static bool operator >(ObjectId lhs, ObjectId rhs) { return lhs.CompareTo(rhs) > 0; } public static bool operator <(ObjectId lhs, ObjectId rhs) { return lhs.CompareTo(rhs) < 0; } public static bool operator <=(ObjectId lhs, ObjectId rhs) { return lhs.CompareTo(rhs) <= 0; } static ObjectId() { _machine = (GetMachineHash() + 10000) & 0xFFFFFF; _increment = new Random().Next(); try { _pid = (short)GetCurrentProcessId(); } catch (SecurityException) { _pid = 0; } } [MethodImpl(MethodImplOptions.NoInlining)] private static int GetCurrentProcessId() { return new Random().Next(0, 5000); } private static int GetMachineHash() { string text = "SOMENAME"; return 0xFFFFFF & text.GetHashCode(); } public static ObjectId NewObjectId() { long num = (long)Math.Floor((DateTime.UtcNow - BsonValue.UnixEpoch).TotalSeconds); return new ObjectId(increment: Interlocked.Increment(ref _increment) & 0xFFFFFF, timestamp: (int)num, machine: _machine, pid: _pid); } } internal class AesEncryption : IDisposable { private Aes _aes; public AesEncryption(string password, byte[] salt) { _aes = Aes.Create(); _aes.Padding = PaddingMode.Zeros; Rfc2898DeriveBytes rfc2898DeriveBytes = new Rfc2898DeriveBytes(password, salt); using (rfc2898DeriveBytes) { _aes.Key = rfc2898DeriveBytes.GetBytes(32); _aes.IV = rfc2898DeriveBytes.GetBytes(16); } } public byte[] Encrypt(byte[] bytes) { using ICryptoTransform transform = _aes.CreateEncryptor(); using MemoryStream memoryStream = new MemoryStream(); using CryptoStream cryptoStream = new CryptoStream(memoryStream, transform, CryptoStreamMode.Write); cryptoStream.Write(bytes, 0, bytes.Length); cryptoStream.FlushFinalBlock(); memoryStream.Position = 0L; byte[] array = new byte[memoryStream.Length]; memoryStream.Read(array, 0, array.Length); return array; } public byte[] Decrypt(byte[] encryptedValue, int offset = 0, int count = -1) { using ICryptoTransform transform = _aes.CreateDecryptor(); using MemoryStream memoryStream = new MemoryStream(); using CryptoStream cryptoStream = new CryptoStream(memoryStream, transform, CryptoStreamMode.Write); cryptoStream.Write(encryptedValue, offset, (count == -1) ? encryptedValue.Length : count); cryptoStream.FlushFinalBlock(); memoryStream.Position = 0L; byte[] array = new byte[memoryStream.Length]; memoryStream.Read(array, 0, array.Length); return array; } public static byte[] HashSHA1(string password) { return SHA1.Create().ComputeHash(Encoding.UTF8.GetBytes(password)); } public static byte[] Salt(int maxLength = 16) { byte[] array = new byte[maxLength]; RandomNumberGenerator randomNumberGenerator = RandomNumberGenerator.Create(); using (randomNumberGenerator) { randomNumberGenerator.GetBytes(array); return array; } } public void Dispose() { if (_aes != null) { _aes.Dispose(); _aes = null; } } } internal class BsonReader { private readonly bool _utcDate; public BsonReader(bool utcDate) { _utcDate = utcDate; } public BsonDocument Deserialize(byte[] bson) { return ReadDocument(new ByteReader(bson)); } public BsonDocument ReadDocument(ByteReader reader) { int num = reader.ReadInt32(); int num2 = reader.Position + num - 5; BsonDocument bsonDocument = new BsonDocument(); while (reader.Position < num2) { string name; BsonValue value = ReadElement(reader, out name); bsonDocument.RawValue[name] = value; } reader.ReadByte(); return bsonDocument; } public BsonArray ReadArray(ByteReader reader) { int num = reader.ReadInt32(); int num2 = reader.Position + num - 5; BsonArray bsonArray = new BsonArray(); while (reader.Position < num2) { string name; BsonValue item = ReadElement(reader, out name); bsonArray.Add(item); } reader.ReadByte(); return bsonArray; } private BsonValue ReadElement(ByteReader reader, out string name) { byte b = reader.ReadByte(); name = reader.ReadCString(); switch (b) { case 1: return reader.ReadDouble(); case 2: return reader.ReadBsonString(); case 3: return ReadDocument(reader); case 4: return ReadArray(reader); case 5: { int count = reader.ReadInt32(); byte b2 = reader.ReadByte(); byte[] array = reader.ReadBytes(count); switch (b2) { case 0: return array; case 4: return new Guid(array); } break; } case 7: return new ObjectId(reader.ReadBytes(12)); case 8: return reader.ReadBoolean(); case 9: { long num = reader.ReadInt64(); switch (num) { case 253402300800000L: return DateTime.MaxValue; case -62135596800000L: return DateTime.MinValue; default: { DateTime unixEpoch = BsonValue.UnixEpoch; DateTime dateTime = unixEpoch.AddMilliseconds(num); return _utcDate ? dateTime : dateTime.ToLocalTime(); } } } case 10: return BsonValue.Null; case 16: return reader.ReadInt32(); case 18: return reader.ReadInt64(); case 19: return reader.ReadDecimal(); case byte.MaxValue: return BsonValue.MinValue; case 127: return BsonValue.MaxValue; } throw new NotSupportedException("BSON type not supported"); } } internal class ByteReader { private readonly byte[] _buffer; private readonly int _length; private int _pos; public int Position { get { return _pos; } set { _pos = value; } } public ByteReader(byte[] buffer) { _buffer = buffer; _length = buffer.Length; _pos = 0; } public void Skip(int length) { _pos += length; } public byte ReadByte() { byte result = _buffer[_pos]; _pos++; return result; } public bool ReadBoolean() { byte num = _buffer[_pos]; _pos++; return num != 0; } public ushort ReadUInt16() { _pos += 2; return BitConverter.ToUInt16(_buffer, _pos - 2); } public uint ReadUInt32() { _pos += 4; return BitConverter.ToUInt32(_buffer, _pos - 4); } public ulong ReadUInt64() { _pos += 8; return BitConverter.ToUInt64(_buffer, _pos - 8); } public short ReadInt16() { _pos += 2; return BitConverter.ToInt16(_buffer, _pos - 2); } public int ReadInt32() { _pos += 4; return BitConverter.ToInt32(_buffer, _pos - 4); } public long ReadInt64() { _pos += 8; return BitConverter.ToInt64(_buffer, _pos - 8); } public float ReadSingle() { _pos += 4; return BitConverter.ToSingle(_buffer, _pos - 4); } public double ReadDouble() { _pos += 8; return BitConverter.ToDouble(_buffer, _pos - 8); } public decimal ReadDecimal() { _pos += 16; int num = BitConverter.ToInt32(_buffer, _pos - 16); int num2 = BitConverter.ToInt32(_buffer, _pos - 12); int num3 = BitConverter.ToInt32(_buffer, _pos - 8); int num4 = BitConverter.ToInt32(_buffer, _pos - 4); return new decimal(new int[4] { num, num2, num3, num4 }); } public byte[] ReadBytes(int count) { byte[] array = new byte[count]; Buffer.BlockCopy(_buffer, _pos, array, 0, count); _pos += count; return array; } public string ReadString() { int num = ReadInt32(); string @string = Encoding.UTF8.GetString(_buffer, _pos, num); _pos += num; return @string; } public string ReadString(int length) { string @string = Encoding.UTF8.GetString(_buffer, _pos, length); _pos += length; return @string; } public string ReadBsonString() { int num = ReadInt32(); string @string = Encoding.UTF8.GetString(_buffer, _pos, num - 1); _pos += num; return @string; } public string ReadCString() { int num = _pos; int num2 = 0; while (true) { if (_buffer[num] == 0) { string @string = Encoding.UTF8.GetString(_buffer, _pos, num2); _pos += num2 + 1; return @string; } if (num > _length) { break; } num++; num2++; } return "_"; } public DateTime ReadDateTime() { return new DateTime(ReadInt64(), DateTimeKind.Utc).ToLocalTime(); } public Guid ReadGuid() { return new Guid(ReadBytes(16)); } public ObjectId ReadObjectId() { return new ObjectId(ReadBytes(12)); } public BsonValue ReadBsonValue(ushort length) { return (BsonType)ReadByte() switch { BsonType.Null => BsonValue.Null, BsonType.Int32 => ReadInt32(), BsonType.Int64 => ReadInt64(), BsonType.Double => ReadDouble(), BsonType.Decimal => ReadDecimal(), BsonType.String => ReadString(length), BsonType.Document => new BsonReader(utcDate: false).ReadDocument(this), BsonType.Array => new BsonReader(utcDate: false).ReadArray(this), BsonType.Binary => ReadBytes(length), BsonType.ObjectId => ReadObjectId(), BsonType.Guid => ReadGuid(), BsonType.Boolean => ReadBoolean(), BsonType.DateTime => ReadDateTime(), BsonType.MinValue => BsonValue.MinValue, BsonType.MaxValue => BsonValue.MaxValue, _ => throw new NotImplementedException(), }; } } internal class AsyncManualResetEvent { private volatile TaskCompletionSource _tcs = new TaskCompletionSource(); public Task WaitAsync() { return _tcs.Task; } public void Set() { _tcs.TrySetResult(result: true); } public void Reset() { TaskCompletionSource tcs; do { tcs = _tcs; } while (tcs.Task.IsCompleted && Interlocked.CompareExchange(ref _tcs, new TaskCompletionSource(), tcs) != tcs); } } internal class BufferSlice { public int Offset { get; } public int Count { get; } public byte[] Array { get; } public byte this[int index] { get { return Array[Offset + index]; } set { Array[Offset + index] = value; } } public BufferSlice(byte[] array, int offset, int count) { Array = array; Offset = offset; Count = count; } public void Clear() { System.Array.Clear(Array, Offset, Count); } public void Clear(int offset, int count) { Constants.ENSURE(offset + count <= Count, "must fit in this page"); System.Array.Clear(Array, Offset + offset, count); } public void Fill(byte value) { for (int i = 0; i < Count; i++) { Array[Offset + i] = value; } } public bool All(byte value) { for (int i = 0; i < Count; i++) { if (Array[Offset + i] != value) { return false; } } return true; } public string ToHex() { StringBuilder stringBuilder = new StringBuilder(); long num = 0L; while (num < Count) { for (int i = 0; i < 32; i++) { if (num >= Count) { break; } stringBuilder.Append(Array[Offset + num].ToString("X2") + " "); num++; } stringBuilder.AppendLine(); } return stringBuilder.ToString(); } public BufferSlice Slice(int offset, int count) { return new BufferSlice(Array, Offset + offset, count); } public byte[] ToArray() { byte[] array = new byte[Count]; Buffer.BlockCopy(Array, Offset, array, 0, Count); return array; } public override string ToString() { return $"Offset: {Offset} - Count: {Count}"; } } public class Collation : IComparer, IComparer, IEqualityComparer { private readonly CompareInfo _compareInfo; public static Collation Default = new Collation(LiteDB.LCID.Current, CompareOptions.IgnoreCase); public static Collation Binary = new Collation(127, CompareOptions.Ordinal); public int LCID { get; } public CultureInfo Culture { get; } public CompareOptions SortOptions { get; } public Collation(string collation) { string[] array = collation.Split(new char[1] { '/' }); string text = array[0]; CompareOptions compareOptions = ((array.Length > 1) ? ((CompareOptions)Enum.Parse(typeof(CompareOptions), array[1])) : CompareOptions.None); LCID = LiteDB.LCID.GetLCID(text); SortOptions = compareOptions; Culture = new CultureInfo(text); _compareInfo = Culture.CompareInfo; } public Collation(int lcid, CompareOptions sortOptions) { LCID = lcid; SortOptions = sortOptions; Culture = LiteDB.LCID.GetCulture(lcid); _compareInfo = Culture.CompareInfo; } public int Compare(string left, string right) { int num = _compareInfo.Compare(left, right, SortOptions); if (num >= 0) { return (num > 0) ? 1 : 0; } return -1; } public int Compare(BsonValue left, BsonValue rigth) { return left.CompareTo(rigth, this); } public bool Equals(BsonValue x, BsonValue y) { return Compare(x, y) == 0; } public int GetHashCode(BsonValue obj) { return obj.GetHashCode(); } public override string ToString() { return Culture.Name + "/" + SortOptions; } } internal class Constants { public const int PAGE_SIZE = 8192; public const int PAGE_HEADER_SIZE = 32; public const int ENCRYPTION_SALT_SIZE = 16; public static int BUFFER_WRITABLE = -1; public static int INDEX_NAME_MAX_LENGTH = 32; public const int MAX_LEVEL_LENGTH = 32; public const int MAX_INDEX_KEY_LENGTH = 1023; public const int MAX_INDEX_LENGTH = 1400; public const int PAGE_FREE_LIST_SLOTS = 5; public const int MAX_DOCUMENT_SIZE = 16683050; public const int MAX_OPEN_TRANSACTIONS = 100; public const int MAX_TRANSACTION_SIZE = 100000; public static int[] MEMORY_SEGMENT_SIZES = new int[5] { 12, 50, 100, 500, 1000 }; public const int VIRTUAL_INDEX_MAX_CACHE = 2000; public const int CONTAINER_SORT_SIZE = 819200; public const int RANDOMIZER_SEED = 0; [DebuggerHidden] [Conditional("DEBUG")] public static void LOG(string message, string category) { } [DebuggerHidden] [Conditional("DEBUG")] public static void LOG(bool conditional, string message, string category) { } [DebuggerHidden] public static void ENSURE(bool conditional, string message = null) { if (!conditional) { if (Debugger.IsAttached) { Debugger.Break(); } throw LiteException.InvalidDatafileState(message); } } [DebuggerHidden] public static void ENSURE(bool conditional, string format, params object[] args) { if (!conditional) { if (Debugger.IsAttached) { Debugger.Break(); } string.Format(CultureInfo.InvariantCulture, format, args); throw LiteException.InvalidDatafileState(format); } } [DebuggerHidden] public static void ENSURE(bool ifTest, bool conditional, string message = null) { if (ifTest && !conditional) { if (Debugger.IsAttached) { Debugger.Break(); } throw LiteException.InvalidDatafileState(message); } } [DebuggerHidden] [Conditional("DEBUG")] public static void DEBUG(bool conditional, string message = null) { if (!conditional) { if (Debugger.IsAttached) { Debugger.Break(); } throw LiteException.InvalidDatafileState(message); } } } internal class StringEncoding { public static Encoding UTF8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); } internal static class ExtendedLengthHelper { public static void ReadLength(byte typeByte, byte lengthByte, out BsonType type, out ushort length) { byte b = (byte)(typeByte & 0x3Fu); byte b2 = lengthByte; byte b3 = (byte)(typeByte & 0xC0u); type = (BsonType)b; length = (ushort)((b3 << 2) | b2); } public static void WriteLength(BsonType type, ushort length, out byte typeByte, out byte lengthByte) { if (length > 1023) { throw new ArgumentOutOfRangeException("length"); } byte b = (byte)type; byte b2 = (byte)length; byte b3 = (byte)((length & 0x300) >> 2); typeByte = (byte)(b3 | b); lengthByte = b2; } } internal static class BufferExtensions { public static int BinaryCompareTo(this byte[] lh, byte[] rh) { if (lh == null) { if (rh != null) { return -1; } return 0; } if (rh == null) { return 1; } int num = 0; int num2 = 0; int num3 = Math.Min(lh.Length, rh.Length); while (num == 0 && num2 < num3) { num = lh[num2].CompareTo(rh[num2]); num2++; } if (num != 0) { if (num >= 0) { return 1; } return -1; } if (num2 == lh.Length) { if (num2 != rh.Length) { return -1; } return 0; } return 1; } public unsafe static bool IsFullZero(this byte[] data) { fixed (byte* ptr = data) { int num = data.Length; int num2 = num % 128; long* ptr2 = (long*)ptr; for (long* ptr3 = (long*)(ptr + num - num2); ptr2 < ptr3; ptr2 += 16) { if ((*ptr2 | ptr2[1] | ptr2[2] | ptr2[3] | ptr2[4] | ptr2[5] | ptr2[6] | ptr2[7] | ptr2[8] | ptr2[9] | ptr2[10] | ptr2[11] | ptr2[12] | ptr2[13] | ptr2[14] | ptr2[15]) != 0L) { return false; } } for (int i = 0; i < num2; i++) { if (data[num - 1 - i] != 0) { return false; } } return true; } } public static byte[] Fill(this byte[] array, byte value, int offset, int count) { for (int i = 0; i < count; i++) { array[i + offset] = value; } return array; } public static string ReadCString(this byte[] bytes, int startIndex, out int bytesCount) { int num = Array.IndexOf(bytes, (byte)0, startIndex); if (num > 0) { bytesCount = num - startIndex; return Encoding.UTF8.GetString(bytes, startIndex, bytesCount); } bytesCount = 0; return null; } public unsafe static void ToBytes(this short value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(short*)ptr = value; } } public unsafe static void ToBytes(this int value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(int*)ptr = value; } } public unsafe static void ToBytes(this long value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(long*)ptr = value; } } public unsafe static void ToBytes(this ushort value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(ushort*)ptr = value; } } public unsafe static void ToBytes(this uint value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(uint*)ptr = value; } } public unsafe static void ToBytes(this ulong value, byte[] array, int startIndex) { fixed (byte* ptr = &array[startIndex]) { *(ulong*)ptr = value; } } public unsafe static void ToBytes(this float value, byte[] array, int startIndex) { (*(uint*)(&value)).ToBytes(array, startIndex); } public unsafe static void ToBytes(this double value, byte[] array, int startIndex) { (*(ulong*)(&value)).ToBytes(array, startIndex); } } internal static class BufferSliceExtensions { public static bool ReadBool(this BufferSlice buffer, int offset) { return buffer.Array[buffer.Offset + offset] != 0; } public static byte ReadByte(this BufferSlice buffer, int offset) { return buffer.Array[buffer.Offset + offset]; } public static short ReadInt16(this BufferSlice buffer, int offset) { return BitConverter.ToInt16(buffer.Array, buffer.Offset + offset); } public static ushort ReadUInt16(this BufferSlice buffer, int offset) { return BitConverter.ToUInt16(buffer.Array, buffer.Offset + offset); } public static int ReadInt32(this BufferSlice buffer, int offset) { return BitConverter.ToInt32(buffer.Array, buffer.Offset + offset); } public static uint ReadUInt32(this BufferSlice buffer, int offset) { return BitConverter.ToUInt32(buffer.Array, buffer.Offset + offset); } public static long ReadInt64(this BufferSlice buffer, int offset) { return BitConverter.ToInt64(buffer.Array, buffer.Offset + offset); } public static double ReadDouble(this BufferSlice buffer, int offset) { return BitConverter.ToDouble(buffer.Array, buffer.Offset + offset); } public static decimal ReadDecimal(this BufferSlice buffer, int offset) { int num = buffer.ReadInt32(offset); int num2 = buffer.ReadInt32(offset + 4); int num3 = buffer.ReadInt32(offset + 8); int num4 = buffer.ReadInt32(offset + 12); return new decimal(new int[4] { num, num2, num3, num4 }); } public static ObjectId ReadObjectId(this BufferSlice buffer, int offset) { return new ObjectId(buffer.Array, buffer.Offset + offset); } public static Guid ReadGuid(this BufferSlice buffer, int offset) { return new Guid(buffer.ReadBytes(offset, 16)); } public static byte[] ReadBytes(this BufferSlice buffer, int offset, int count) { byte[] array = new byte[count]; Buffer.BlockCopy(buffer.Array, buffer.Offset + offset, array, 0, count); return array; } public static DateTime ReadDateTime(this BufferSlice buffer, int offset) { long num = buffer.ReadInt64(offset); return num switch { 0L => DateTime.MinValue, 3155378975999999999L => DateTime.MaxValue, _ => new DateTime(num, DateTimeKind.Utc), }; } public static PageAddress ReadPageAddress(this BufferSlice buffer, int offset) { return new PageAddress(buffer.ReadUInt32(offset), buffer[offset + 4]); } public static string ReadString(this BufferSlice buffer, int offset, int count) { return Encoding.UTF8.GetString(buffer.Array, buffer.Offset + offset, count); } public static string ReadCString(this BufferSlice buffer, int offset, out int length) { length = buffer.Count - buffer.Offset - offset; for (int i = offset + buffer.Offset; i < buffer.Count; i++) { if (buffer[i] == 0) { length = i - buffer.Offset - offset + 1; break; } } return Encoding.UTF8.GetString(buffer.Array, buffer.Offset + offset, length - 1); } public static BsonValue ReadIndexKey(this BufferSlice buffer, int offset) { ExtendedLengthHelper.ReadLength(buffer[offset++], buffer[offset], out var type, out var length); switch (type) { case BsonType.Null: return BsonValue.Null; case BsonType.Int32: return buffer.ReadInt32(offset); case BsonType.Int64: return buffer.ReadInt64(offset); case BsonType.Double: return buffer.ReadDouble(offset); case BsonType.Decimal: return buffer.ReadDecimal(offset); case BsonType.String: offset++; return buffer.ReadString(offset, length); case BsonType.Document: { using BufferReader bufferReader2 = new BufferReader(buffer); bufferReader2.Skip(offset); return bufferReader2.ReadDocument().GetValue(); } case BsonType.Array: { using BufferReader bufferReader = new BufferReader(buffer); bufferReader.Skip(offset); return bufferReader.ReadArray().GetValue(); } case BsonType.Binary: offset++; return buffer.ReadBytes(offset, length); case BsonType.ObjectId: return buffer.ReadObjectId(offset); case BsonType.Guid: return buffer.ReadGuid(offset); case BsonType.Boolean: return buffer[offset] != 0; case BsonType.DateTime: return buffer.ReadDateTime(offset); case BsonType.MinValue: return BsonValue.MinValue; case BsonType.MaxValue: return BsonValue.MaxValue; default: throw new NotImplementedException(); } } public static void Write(this BufferSlice buffer, bool value, int offset) { buffer.Array[buffer.Offset + offset] = (value ? ((byte)1) : ((byte)0)); } public static void Write(this BufferSlice buffer, byte value, int offset) { buffer.Array[buffer.Offset + offset] = value; } public static void Write(this BufferSlice buffer, short value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, ushort value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, int value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, uint value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, long value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, double value, int offset) { value.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, decimal value, int offset) { int[] bits = decimal.GetBits(value); buffer.Write(bits[0], offset); buffer.Write(bits[1], offset + 4); buffer.Write(bits[2], offset + 8); buffer.Write(bits[3], offset + 12); } public static void Write(this BufferSlice buffer, DateTime value, int offset) { value.ToUniversalTime().Ticks.ToBytes(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, PageAddress value, int offset) { value.PageID.ToBytes(buffer.Array, buffer.Offset + offset); buffer[offset + 4] = value.Index; } public static void Write(this BufferSlice buffer, Guid value, int offset) { buffer.Write(value.ToByteArray(), offset); } public static void Write(this BufferSlice buffer, ObjectId value, int offset) { value.ToByteArray(buffer.Array, buffer.Offset + offset); } public static void Write(this BufferSlice buffer, byte[] value, int offset) { Buffer.BlockCopy(value, 0, buffer.Array, buffer.Offset + offset, value.Length); } public static void Write(this BufferSlice buffer, string value, int offset) { Encoding.UTF8.GetBytes(value, 0, value.Length, buffer.Array, buffer.Offset + offset); } public static void WriteIndexKey(this BufferSlice buffer, BsonValue value, int offset) { if (value.IsString) { string asString = value.AsString; ushort length = (ushort)Encoding.UTF8.GetByteCount(asString); ExtendedLengthHelper.WriteLength(BsonType.String, length, out var typeByte, out var lengthByte); buffer[offset++] = typeByte; buffer[offset++] = lengthByte; buffer.Write(asString, offset); return; } if (value.IsBinary) { byte[] asBinary = value.AsBinary; ExtendedLengthHelper.WriteLength(BsonType.Binary, (ushort)asBinary.Length, out var typeByte2, out var lengthByte2); buffer[offset++] = typeByte2; buffer[offset++] = lengthByte2; buffer.Write(asBinary, offset); return; } buffer[offset++] = (byte)value.Type; switch (value.Type) { case BsonType.Int32: buffer.Write(value.AsInt32, offset); break; case BsonType.Int64: buffer.Write(value.AsInt64, offset); break; case BsonType.Double: buffer.Write(value.AsDouble, offset); break; case BsonType.Decimal: buffer.Write(value.AsDecimal, offset); break; case BsonType.Document: { using BufferWriter bufferWriter2 = new BufferWriter(buffer); bufferWriter2.Skip(offset); bufferWriter2.WriteDocument(value.AsDocument, recalc: true); break; } case BsonType.Array: { using BufferWriter bufferWriter = new BufferWriter(buffer); bufferWriter.Skip(offset); bufferWriter.WriteArray(value.AsArray, recalc: true); break; } case BsonType.ObjectId: buffer.Write(value.AsObjectId, offset); break; case BsonType.Guid: buffer.Write(value.AsGuid, offset); break; case BsonType.Boolean: buffer[offset] = (value.AsBoolean ? ((byte)1) : ((byte)0)); break; case BsonType.DateTime: buffer.Write(value.AsDateTime, offset); break; default: throw new NotImplementedException(); case BsonType.MinValue: case BsonType.Null: case BsonType.MaxValue: break; } } } internal static class DateExtensions { public static DateTime Truncate(this DateTime dt) { if (dt == DateTime.MaxValue || dt == DateTime.MinValue) { return dt; } return new DateTime(dt.Year, dt.Month, dt.Day, dt.Hour, dt.Minute, dt.Second, dt.Millisecond, dt.Kind); } public static int MonthDifference(this DateTime startDate, DateTime endDate) { int num = endDate.Month + endDate.Year * 12 - (startDate.Month + startDate.Year * 12); double num2 = (endDate - endDate.AddMonths(1)).Days; return Convert.ToInt32(Math.Truncate((double)num + (double)(startDate.Day - endDate.Day) / num2)); } public static int YearDifference(this DateTime startDate, DateTime endDate) { int num = endDate.Year - startDate.Year; if (startDate.Month == endDate.Month && endDate.Day < startDate.Day) { num--; } else if (endDate.Month < startDate.Month) { num--; } return num; } } internal static class DictionaryExtensions { public static T GetOrDefault(this IDictionary dict, K key, T defaultValue = default(T)) { if (dict.TryGetValue(key, out var value)) { return value; } return defaultValue; } public static T GetOrAdd(this IDictionary dict, K key, Func valueFactoy) { if (!dict.TryGetValue(key, out var value)) { value = valueFactoy(key); dict.Add(key, value); } return value; } public static void ParseKeyValue(this IDictionary dict, string connectionString) { int position = 0; while (position < connectionString.Length) { EatWhitespace(); string key = ReadKey(); EatWhitespace(); string value = ReadValue(); dict[key] = value; } void EatWhitespace() { for (; position < connectionString.Length && (connectionString[position] == ' ' || connectionString[position] == '\t' || connectionString[position] == '\f'); position++) { } } string ReadKey() { StringBuilder stringBuilder2 = new StringBuilder(); for (; position < connectionString.Length; position++) { char c3 = connectionString[position]; if (c3 == '=') { position++; return stringBuilder2.ToString().Trim(); } stringBuilder2.Append(c3); } return stringBuilder2.ToString().Trim(); } string ReadValue() { StringBuilder stringBuilder = new StringBuilder(); char c = ((connectionString[position] == '"') ? '"' : ((connectionString[position] == '\'') ? '\'' : ' ')); if (c != ' ') { position++; } for (; position < connectionString.Length; position++) { char c2 = connectionString[position]; if (c == ' ') { if (c2 == ';') { position++; return stringBuilder.ToString().Trim(); } } else if (c != ' ' && c2 == c) { if (connectionString[position - 1] != '\\') { position++; EatWhitespace(); if (position < connectionString.Length && connectionString[position] == ';') { position++; } return stringBuilder.ToString(); } stringBuilder.Length--; } stringBuilder.Append(c2); } return stringBuilder.ToString().Trim(); } } public static T GetValue(this Dictionary dict, string key, T defaultValue = default(T)) { try { if (!dict.TryGetValue(key, out var value)) { return defaultValue; } if (typeof(T) == typeof(TimeSpan)) { if (Regex.IsMatch(value, "^\\d+$", RegexOptions.Compiled)) { return (T)(object)TimeSpan.FromSeconds(Convert.ToInt32(value)); } return (T)(object)TimeSpan.Parse(value); } if (typeof(T).GetTypeInfo().IsEnum) { return (T)Enum.Parse(typeof(T), value, ignoreCase: true); } return (T)Convert.ChangeType(value, typeof(T)); } catch (Exception) { throw new LiteException(0, "Invalid connection string value type for `" + key + "`"); } } public static long GetFileSize(this Dictionary dict, string key, long defaultValue) { string value = dict.GetValue(key); if (value == null) { return defaultValue; } Match match = Regex.Match(value, "^(\\d+)\\s*([tgmk])?(b|byte|bytes)?$", RegexOptions.IgnoreCase); if (!match.Success) { return 0L; } long num = Convert.ToInt64(match.Groups[1].Value); string text = match.Groups[2].Value.ToLower(); switch (text) { default: if (text.Length != 0) { break; } return num; case "t": return num * 1024 * 1024 * 1024 * 1024; case "g": return num * 1024 * 1024 * 1024; case "m": return num * 1024 * 1024; case "k": return num * 1024; case null: break; } return 0L; } } internal static class ExpressionExtensions { private static readonly Regex _removeSelect = new Regex("\\.Select\\s*\\(\\s*\\w+\\s*=>\\s*\\w+\\.", RegexOptions.Compiled); private static readonly Regex _removeList = new Regex("\\.get_Item\\(\\d+\\)", RegexOptions.Compiled); private static readonly Regex _removeArray = new Regex("\\[\\d+\\]", RegexOptions.Compiled); public static string GetPath(this Expression expr) { while (expr.NodeType == ExpressionType.Convert || expr.NodeType == ExpressionType.ConvertChecked) { expr = ((UnaryExpression)expr).Operand; } while (expr.NodeType == ExpressionType.Lambda && ((LambdaExpression)expr).Body is UnaryExpression unaryExpression) { expr = unaryExpression.Operand; } string text = expr.ToString(); int num = text.IndexOf('.'); string input = ((num < 0) ? text : text.Substring(num + 1).TrimEnd(new char[1] { ')' })); input = _removeList.Replace(input, ""); input = _removeArray.Replace(input, ""); return _removeSelect.Replace(input, ".").Replace(")", ""); } } internal static class HashSetExtensions { public static HashSet AddRange(this HashSet hash, IEnumerable items) { if (items == null) { return hash; } foreach (T item in items) { hash.Add(item); } return hash; } } internal static class IOExceptionExtensions { private const int ERROR_SHARING_VIOLATION = 32; private const int ERROR_LOCK_VIOLATION = 33; public static bool IsLocked(this IOException ex) { int num = Marshal.GetHRForException(ex) & 0xFFFF; if (num != 32) { return num == 33; } return true; } public static void WaitIfLocked(this IOException ex, int timerInMilliseconds) { if (ex.IsLocked()) { if (timerInMilliseconds > 0) { Task.Delay(timerInMilliseconds).Wait(); } return; } throw ex; } } internal static class LinqExtensions { public static IEnumerable> Batch(this IEnumerable source, int batchSize) { using IEnumerator enumerator = source.GetEnumerator(); while (enumerator.MoveNext()) { yield return YieldBatchElements(enumerator, batchSize - 1); } } private static IEnumerable YieldBatchElements(IEnumerator source, int batchSize) { yield return source.Current; for (int i = 0; i < batchSize; i++) { if (!source.MoveNext()) { break; } yield return source.Current; } } public static IEnumerable DistinctBy(this IEnumerable source, Func keySelector, IEqualityComparer comparer) { if (source == null) { throw new ArgumentNullException("source"); } if (keySelector == null) { throw new ArgumentNullException("keySelector"); } return _(); IEnumerable _() { HashSet knownKeys = new HashSet(comparer); foreach (TSource item in source) { if (knownKeys.Add(keySelector(item))) { yield return item; } } } } public static IEnumerable> IsLast(this IEnumerable source) where T : class { T val = null; foreach (T item in source) { if (val != null) { yield return new LastItem { Item = val, IsLast = false }; } val = item; } if (val != null) { yield return new LastItem { Item = val, IsLast = true }; } } } internal struct LastItem { public T Item { get; set; } public bool IsLast { get; set; } } internal static class StreamExtensions { public static void FlushToDisk(this Stream stream) { if (stream is FileStream fileStream) { fileStream.Flush(flushToDisk: true); } else { stream.Flush(); } } } internal static class StringExtensions { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNullOrWhiteSpace(this string str) { return string.IsNullOrWhiteSpace(str); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNullOrEmpty(this string str) { return string.IsNullOrEmpty(str); } public static bool IsWord(this string str) { if (string.IsNullOrWhiteSpace(str)) { return false; } for (int i = 0; i < str.Length; i++) { if (!Tokenizer.IsWordChar(str[i], i == 0)) { return false; } } return true; } public static string Sha1(this string value) { byte[] bytes = Encoding.UTF8.GetBytes(value); using SHA1 sHA = SHA1.Create(); byte[] array = sHA.ComputeHash(bytes); StringBuilder stringBuilder = new StringBuilder(); byte[] array2 = array; foreach (byte b in array2) { stringBuilder.Append(b.ToString("X2")); } return stringBuilder.ToString(); } public static bool SqlLike(this string str, string pattern, Collation collation) { bool flag = true; bool flag2 = false; bool flag3 = false; bool flag4 = false; bool flag5 = false; bool flag6 = false; int num = -1; int i = 0; char c = '\0'; for (int j = 0; j < str.Length; j++) { char c2 = str[j]; if (i < pattern.Length) { c = pattern[i]; if (!flag2 && c == '%') { num = i; flag2 = true; for (; i < pattern.Length && pattern[i] == '%'; i++) { } c = ((i < pattern.Length) ? pattern[i] : '\0'); } else if (c == '_') { flag3 = true; i++; } } if (flag2) { if (collation.Compare(c2.ToString(), c.ToString()) == 0) { flag2 = false; i++; } } else if (flag3) { flag3 = false; } else if (flag4 || flag5) { if (flag4) { if (num < 0) { flag = false; break; } i = num; } flag5 = (flag4 = false); } else if (collation.Compare(c2.ToString(), c.ToString()) == 0) { i++; } else { if (num < 0) { flag = false; break; } int num2 = i - num - 1; j -= num2; i = num; } } flag6 = i >= pattern.Length; if (flag && !flag6) { bool flag7 = true; for (int k = i; k < pattern.Length; k++) { if (pattern[k] != '%') { flag7 = false; break; } } if (flag7) { flag6 = true; } } return flag && flag6; } public static string SqlLikeStartsWith(this string str, out bool hasMore) { int i = 0; int length = str.Length; char c = '\0'; for (; i < length; i++) { c = str[i]; if (c == '%' || c == '_') { break; } } hasMore = i != length && i != length - 1; return str.Substring(0, i); } } internal static class TypeInfoExtensions { public static bool IsAnonymousType(this Type type) { if (type.FullName.Contains("AnonymousType")) { return type.GetTypeInfo().GetCustomAttributes(typeof(CompilerGeneratedAttribute), inherit: false).Any(); } return false; } public static bool IsEnumerable(this Type type) { if (type != typeof(string)) { return typeof(IEnumerable).IsAssignableFrom(type); } return false; } } internal static class FileHelper { public static string GetSuffixFile(string filename, string suffix = "-temp", bool checkIfExists = true) { int num = 0; string text = Path.Combine(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename) + suffix + Path.GetExtension(filename)); while (checkIfExists && File.Exists(text)) { string? directoryName = Path.GetDirectoryName(filename); string[] obj = new string[5] { Path.GetFileNameWithoutExtension(filename), suffix, "-", null, null }; int num2 = ++num; obj[3] = num2.ToString(); obj[4] = Path.GetExtension(filename); text = Path.Combine(directoryName, string.Concat(obj)); } return text; } public static string GetLogFile(string filename) { return GetSuffixFile(filename, "-log", checkIfExists: false); } public static string GetTempFile(string filename) { return GetSuffixFile(filename, "-tmp", checkIfExists: false); } public static bool IsFileLocked(string filename) { FileStream fileStream = null; FileInfo fileInfo = new FileInfo(filename); try { fileStream = fileInfo.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None); } catch (IOException ex) { return ex.IsLocked(); } finally { fileStream?.Dispose(); } return false; } public static bool TryExec(int timeout, Action action) { DateTime dateTime = DateTime.UtcNow.AddSeconds(timeout); do { try { action(); return true; } catch (IOException ex) { ex.WaitIfLocked(25); } } while (DateTime.UtcNow < dateTime); return false; } public static void Exec(int timeout, Action action) { DateTime dateTime = DateTime.UtcNow.AddSeconds(timeout); IOException ex2; do { try { action(); return; } catch (IOException ex) { ex2 = ex; ex.WaitIfLocked(25); } } while (DateTime.UtcNow < dateTime); throw ex2; } public static long ParseFileSize(string size) { Match match = Regex.Match(size, "^(\\d+)\\s*([tgmk])?(b|byte|bytes)?$", RegexOptions.IgnoreCase); if (!match.Success) { return 0L; } long num = Convert.ToInt64(match.Groups[1].Value); string text = match.Groups[2].Value.ToLower(); switch (text) { default: if (text.Length != 0) { break; } return num; case "t": return num * 1024 * 1024 * 1024 * 1024; case "g": return num * 1024 * 1024 * 1024; case "m": return num * 1024 * 1024; case "k": return num * 1024; case null: break; } return 0L; } public static string FormatFileSize(long byteCount) { string[] array = new string[5] { "B", "KB", "MB", "GB", "TB" }; if (byteCount == 0L) { return "0" + array[0]; } long num = Math.Abs(byteCount); long num2 = Convert.ToInt64(Math.Floor(Math.Log(num, 1024.0))); double num3 = Math.Round((double)num / Math.Pow(1024.0, num2), 1); return (double)Math.Sign(byteCount) * num3 + array[num2]; } } internal class LCID { private static readonly IDictionary _mappings = new Dictionary { [1] = "ar", [2] = "bg", [3] = "ca", [4] = "zh-Hans", [5] = "cs", [6] = "da", [7] = "de", [8] = "el", [9] = "en", [10] = "es", [11] = "fi", [12] = "fr", [13] = "he", [14] = "hu", [15] = "is", [16] = "it", [17] = "ja", [18] = "ko", [19] = "nl", [20] = "no", [21] = "pl", [22] = "pt", [23] = "rm", [24] = "ro", [25] = "ru", [26] = "hr", [27] = "sk", [28] = "sq", [29] = "sv", [30] = "th", [31] = "tr", [32] = "ur", [33] = "id", [34] = "uk", [35] = "be", [36] = "sl", [37] = "et", [38] = "lv", [39] = "lt", [40] = "tg", [41] = "fa", [42] = "vi", [43] = "hy", [44] = "az", [45] = "eu", [46] = "hsb", [47] = "mk", [48] = "st", [49] = "ts", [50] = "tn", [51] = "ve", [52] = "xh", [53] = "zu", [54] = "af", [55] = "ka", [56] = "fo", [57] = "hi", [58] = "mt", [59] = "se", [60] = "ga", [61] = "yi", [62] = "ms", [63] = "kk", [64] = "ky", [65] = "sw", [66] = "tk", [67] = "uz", [68] = "tt", [69] = "bn", [70] = "pa", [71] = "gu", [72] = "or", [73] = "ta", [74] = "te", [75] = "kn", [76] = "ml", [77] = "as", [78] = "mr", [79] = "sa", [80] = "mn", [81] = "bo", [82] = "cy", [83] = "km", [84] = "lo", [85] = "my", [86] = "gl", [87] = "kok", [88] = "mni", [89] = "sd", [90] = "syr", [91] = "si", [92] = "chr", [93] = "iu", [94] = "am", [95] = "tzm", [96] = "ks", [97] = "ne", [98] = "fy", [99] = "ps", [100] = "fil", [101] = "dv", [102] = "bin", [103] = "ff", [104] = "ha", [105] = "ibb", [106] = "yo", [107] = "quz", [108] = "nso", [109] = "ba", [110] = "lb", [111] = "kl", [112] = "ig", [113] = "kr", [114] = "om", [115] = "ti", [116] = "gn", [117] = "haw", [118] = "la", [119] = "so", [120] = "ii", [121] = "pap", [122] = "arn", [124] = "moh", [126] = "br", [127] = "", [128] = "ug", [129] = "mi", [130] = "oc", [131] = "co", [132] = "gsw", [133] = "sah", [134] = "quc", [135] = "rw", [136] = "wo", [140] = "prs", [145] = "gd", [146] = "ku", [1025] = "ar-SA", [1026] = "bg-BG", [1027] = "ca-ES", [1028] = "zh-TW", [1029] = "cs-CZ", [1030] = "da-DK", [1031] = "de-DE", [1032] = "el-GR", [1033] = "en-US", [1035] = "fi-FI", [1036] = "fr-FR", [1037] = "he-IL", [1038] = "hu-HU", [1039] = "is-IS", [1040] = "it-IT", [1041] = "ja-JP", [1042] = "ko-KR", [1043] = "nl-NL", [1044] = "nb-NO", [1045] = "pl-PL", [1046] = "pt-BR", [1047] = "rm-CH", [1048] = "ro-RO", [1049] = "ru-RU", [1050] = "hr-HR", [1051] = "sk-SK", [1052] = "sq-AL", [1053] = "sv-SE", [1054] = "th-TH", [1055] = "tr-TR", [1056] = "ur-PK", [1057] = "id-ID", [1058] = "uk-UA", [1059] = "be-BY", [1060] = "sl-SI", [1061] = "et-EE", [1062] = "lv-LV", [1063] = "lt-LT", [1064] = "tg-Cyrl-TJ", [1065] = "fa-IR", [1066] = "vi-VN", [1067] = "hy-AM", [1068] = "az-Latn-AZ", [1069] = "eu-ES", [1070] = "hsb-DE", [1071] = "mk-MK", [1072] = "st-ZA", [1073] = "ts-ZA", [1074] = "tn-ZA", [1075] = "ve-ZA", [1076] = "xh-ZA", [1077] = "zu-ZA", [1078] = "af-ZA", [1079] = "ka-GE", [1080] = "fo-FO", [1081] = "hi-IN", [1082] = "mt-MT", [1083] = "se-NO", [1085] = "yi-001", [1086] = "ms-MY", [1087] = "kk-KZ", [1088] = "ky-KG", [1089] = "sw-KE", [1090] = "tk-TM", [1091] = "uz-Latn-UZ", [1092] = "tt-RU", [1093] = "bn-IN", [1094] = "pa-IN", [1095] = "gu-IN", [1096] = "or-IN", [1097] = "ta-IN", [1098] = "te-IN", [1099] = "kn-IN", [1100] = "ml-IN", [1101] = "as-IN", [1102] = "mr-IN", [1103] = "sa-IN", [1104] = "mn-MN", [1105] = "bo-CN", [1106] = "cy-GB", [1107] = "km-KH", [1108] = "lo-LA", [1109] = "my-MM", [1110] = "gl-ES", [1111] = "kok-IN", [1112] = "mni-IN", [1113] = "sd-Deva-IN", [1114] = "syr-SY", [1115] = "si-LK", [1116] = "chr-Cher-US", [1117] = "iu-Cans-CA", [1118] = "am-ET", [1119] = "tzm-Arab-MA", [1120] = "ks-Arab", [1121] = "ne-NP", [1122] = "fy-NL", [1123] = "ps-AF", [1124] = "fil-PH", [1125] = "dv-MV", [1126] = "bin-NG", [1127] = "ff-NG", [1128] = "ha-Latn-NG", [1129] = "ibb-NG", [1130] = "yo-NG", [1131] = "quz-BO", [1132] = "nso-ZA", [1133] = "ba-RU", [1134] = "lb-LU", [1135] = "kl-GL", [1136] = "ig-NG", [1137] = "kr-NG", [1138] = "om-ET", [1139] = "ti-ET", [1140] = "gn-PY", [1141] = "haw-US", [1142] = "la-001", [1143] = "so-SO", [1144] = "ii-CN", [1145] = "pap-029", [1146] = "arn-CL", [1148] = "moh-CA", [1150] = "br-FR", [1152] = "ug-CN", [1153] = "mi-NZ", [1154] = "oc-FR", [1155] = "co-FR", [1156] = "gsw-FR", [1157] = "sah-RU", [1158] = "quc-Latn-GT", [1159] = "rw-RW", [1160] = "wo-SN", [1164] = "prs-AF", [1169] = "gd-GB", [1170] = "ku-Arab-IQ", [2049] = "ar-IQ", [2051] = "ca-ES-valencia", [2052] = "zh-CN", [2055] = "de-CH", [2057] = "en-GB", [2058] = "es-MX", [2060] = "fr-BE", [2064] = "it-CH", [2067] = "nl-BE", [2068] = "nn-NO", [2070] = "pt-PT", [2072] = "ro-MD", [2073] = "ru-MD", [2077] = "sv-FI", [2080] = "ur-IN", [2092] = "az-Cyrl-AZ", [2094] = "dsb-DE", [2098] = "tn-BW", [2107] = "se-SE", [2108] = "ga-IE", [2110] = "ms-BN", [2115] = "uz-Cyrl-UZ", [2117] = "bn-BD", [2118] = "pa-Arab-PK", [2121] = "ta-LK", [2128] = "mn-Mong-CN", [2137] = "sd-Arab-PK", [2141] = "iu-Latn-CA", [2143] = "tzm-Latn-DZ", [2144] = "ks-Deva-IN", [2145] = "ne-IN", [2151] = "ff-Latn-SN", [2155] = "quz-EC", [2163] = "ti-ER", [3073] = "ar-EG", [3076] = "zh-HK", [3079] = "de-AT", [3081] = "en-AU", [3082] = "es-ES", [3084] = "fr-CA", [3131] = "se-FI", [3152] = "mn-Mong-MN", [3153] = "dz-BT", [3179] = "quz-PE", [4097] = "ar-LY", [4100] = "zh-SG", [4103] = "de-LU", [4105] = "en-CA", [4106] = "es-GT", [4108] = "fr-CH", [4122] = "hr-BA", [4155] = "smj-NO", [4191] = "tzm-Tfng-MA", [5121] = "ar-DZ", [5124] = "zh-MO", [5127] = "de-LI", [5129] = "en-NZ", [5130] = "es-CR", [5132] = "fr-LU", [5146] = "bs-Latn-BA", [5179] = "smj-SE", [6145] = "ar-MA", [6153] = "en-IE", [6154] = "es-PA", [6156] = "fr-MC", [6170] = "sr-Latn-BA", [6203] = "sma-NO", [7169] = "ar-TN", [7177] = "en-ZA", [7178] = "es-DO", [7180] = "fr-029", [7194] = "sr-Cyrl-BA", [7227] = "sma-SE", [8193] = "ar-OM", [8201] = "en-JM", [8202] = "es-VE", [8204] = "fr-RE", [8218] = "bs-Cyrl-BA", [8251] = "sms-FI", [9217] = "ar-YE", [9225] = "en-029", [9226] = "es-CO", [9228] = "fr-CD", [9242] = "sr-Latn-RS", [9275] = "smn-FI", [10241] = "ar-SY", [10249] = "en-BZ", [10250] = "es-PE", [10252] = "fr-SN", [10266] = "sr-Cyrl-RS", [11265] = "ar-JO", [11273] = "en-TT", [11274] = "es-AR", [11276] = "fr-CM", [11290] = "sr-Latn-ME", [12289] = "ar-LB", [12297] = "en-ZW", [12298] = "es-EC", [12300] = "fr-CI", [12314] = "sr-Cyrl-ME", [13313] = "ar-KW", [13321] = "en-PH", [13322] = "es-CL", [13324] = "fr-ML", [14337] = "ar-AE", [14345] = "en-ID", [14346] = "es-UY", [14348] = "fr-MA", [15361] = "ar-BH", [15369] = "en-HK", [15370] = "es-PY", [15372] = "fr-HT", [16385] = "ar-QA", [16393] = "en-IN", [16394] = "es-BO", [17417] = "en-MY", [17418] = "es-SV", [18441] = "en-SG", [18442] = "es-HN", [19466] = "es-NI", [20490] = "es-PR", [21514] = "es-US", [22538] = "es-419", [23562] = "es-CU", [25626] = "bs-Cyrl", [26650] = "bs-Latn", [27674] = "sr-Cyrl", [28698] = "sr-Latn", [28731] = "smn", [29740] = "az-Cyrl", [29755] = "sms", [30724] = "zh", [30740] = "nn", [30746] = "bs", [30764] = "az-Latn", [30779] = "sma", [30787] = "uz-Cyrl", [30800] = "mn-Cyrl", [30813] = "iu-Cans", [30815] = "tzm-Tfng", [31748] = "zh-Hant", [31764] = "nb", [31770] = "sr", [31784] = "tg-Cyrl", [31790] = "dsb", [31803] = "smj", [31811] = "uz-Latn", [31814] = "pa-Arab", [31824] = "mn-Mong", [31833] = "sd-Arab", [31836] = "chr-Cher", [31837] = "iu-Latn", [31839] = "tzm-Latn", [31847] = "ff-Latn", [31848] = "ha-Latn", [31878] = "quc-Latn", [31890] = "ku-Arab" }; public static int Current { get { string current = CultureInfo.CurrentCulture.Name; KeyValuePair keyValuePair = _mappings.FirstOrDefault((KeyValuePair x) => x.Value == current); if (keyValuePair.Key == 0) { return 127; } return keyValuePair.Key; } } public static CultureInfo GetCulture(int lcid) { if (_mappings.TryGetValue(lcid, out var value)) { return new CultureInfo(value); } throw new ArgumentException("Invalid LCID code"); } public static int GetLCID(string culture) { foreach (KeyValuePair mapping in _mappings) { if (mapping.Value == culture) { return mapping.Key; } } throw new LiteException(0, "Invalid culture name"); } } public class LiteException : Exception { public const int FILE_NOT_FOUND = 101; public const int DATABASE_SHUTDOWN = 102; public const int INVALID_DATABASE = 103; public const int FILE_SIZE_EXCEEDED = 105; public const int COLLECTION_LIMIT_EXCEEDED = 106; public const int INDEX_DROP_ID = 108; public const int INDEX_DUPLICATE_KEY = 110; public const int INVALID_INDEX_KEY = 111; public const int INDEX_NOT_FOUND = 112; public const int INVALID_DBREF = 113; public const int LOCK_TIMEOUT = 120; public const int INVALID_COMMAND = 121; public const int ALREADY_EXISTS_COLLECTION_NAME = 122; public const int ALREADY_OPEN_DATAFILE = 124; public const int INVALID_TRANSACTION_STATE = 126; public const int INDEX_NAME_LIMIT_EXCEEDED = 128; public const int INVALID_INDEX_NAME = 129; public const int INVALID_COLLECTION_NAME = 130; public const int TEMP_ENGINE_ALREADY_DEFINED = 131; public const int INVALID_EXPRESSION_TYPE = 132; public const int COLLECTION_NOT_FOUND = 133; public const int COLLECTION_ALREADY_EXIST = 134; public const int INDEX_ALREADY_EXIST = 135; public const int INVALID_UPDATE_FIELD = 136; public const int ENGINE_DISPOSED = 137; public const int INVALID_FORMAT = 200; public const int DOCUMENT_MAX_DEPTH = 201; public const int INVALID_CTOR = 202; public const int UNEXPECTED_TOKEN = 203; public const int INVALID_DATA_TYPE = 204; public const int PROPERTY_NOT_MAPPED = 206; public const int INVALID_TYPED_NAME = 207; public const int PROPERTY_READ_WRITE = 209; public const int INITIALSIZE_CRYPTO_NOT_SUPPORTED = 210; public const int INVALID_INITIALSIZE = 211; public const int INVALID_NULL_CHAR_STRING = 212; public const int INVALID_FREE_SPACE_PAGE = 213; public const int DATA_TYPE_NOT_ASSIGNABLE = 214; public const int AVOID_USE_OF_PROCESS = 215; public const int NOT_ENCRYPTED = 216; public const int INVALID_PASSWORD = 217; public const int ILLEGAL_DESERIALIZATION_TYPE = 218; public const int INVALID_DATAFILE_STATE = 999; public int ErrorCode { get; private set; } public long Position { get; private set; } public bool IsCritical => ErrorCode >= 900; public LiteException(int code, string message) : base(message) { ErrorCode = code; } internal LiteException(int code, string message, params object[] args) : base(string.Format(message, args)) { ErrorCode = code; } internal LiteException(int code, Exception inner, string message, params object[] args) : base(string.Format(message, args), inner) { ErrorCode = code; } internal static LiteException FileNotFound(object fileId) { return new LiteException(101, "File '{0}' not found.", fileId); } internal static LiteException DatabaseShutdown() { return new LiteException(102, "Database is in shutdown process."); } internal static LiteException InvalidDatabase() { return new LiteException(103, "File is not a valid LiteDB database format or contains a invalid password."); } internal static LiteException FileSizeExceeded(long limit) { return new LiteException(105, "Database size exceeds limit of {0}.", FileHelper.FormatFileSize(limit)); } internal static LiteException CollectionLimitExceeded(int limit) { return new LiteException(106, "This database exceeded the maximum limit of collection names size: {0} bytes", limit); } internal static LiteException IndexNameLimitExceeded(int limit) { return new LiteException(128, "This collection exceeded the maximum limit of indexes names/expression size: {0} bytes", limit); } internal static LiteException InvalidIndexName(string name, string collection, string reason) { return new LiteException(129, "Invalid index name '{0}' on collection '{1}': {2}", name, collection, reason); } internal static LiteException InvalidCollectionName(string name, string reason) { return new LiteException(130, "Invalid collection name '{0}': {1}", name, reason); } internal static LiteException IndexDropId() { return new LiteException(108, "Primary key index '_id' can't be dropped."); } internal static LiteException TempEngineAlreadyDefined() { return new LiteException(131, "Temporary engine already defined or auto created."); } internal static LiteException CollectionNotFound(string key) { return new LiteException(133, "Collection not found: '{0}'", key); } internal static LiteException InvalidExpressionType(BsonExpression expr, BsonExpressionType type) { return new LiteException(132, "Expression '{0}' must be a {1} type.", expr.Source, type); } internal static LiteException InvalidExpressionTypePredicate(BsonExpression expr) { return new LiteException(132, "Expression '{0}' are not supported as predicate expression.", expr.Source); } internal static LiteException CollectionAlreadyExist(string key) { return new LiteException(134, "Collection already exist: '{0}'", key); } internal static LiteException IndexAlreadyExist(string name) { return new LiteException(135, "Index name '{0}' already exist with a differnt expression. Try drop index first.", name); } internal static LiteException InvalidUpdateField(string field) { return new LiteException(136, "'{0}' can't be modified in UPDATE command.", field); } internal static LiteException IndexDuplicateKey(string field, BsonValue key) { return new LiteException(110, "Cannot insert duplicate key in unique index '{0}'. The duplicate value is '{1}'.", field, key); } internal static LiteException InvalidIndexKey(string text) { return new LiteException(111, text); } internal static LiteException IndexNotFound(string name) { return new LiteException(112, "Index not found '{0}'.", name); } internal static LiteException LockTimeout(string mode, TimeSpan ts) { return new LiteException(120, "Database lock timeout when entering in {0} mode after {1}", mode, ts.ToString()); } internal static LiteException LockTimeout(string mode, string collection, TimeSpan ts) { return new LiteException(120, "Collection '{0}' lock timeout when entering in {1} mode after {2}", collection, mode, ts.ToString()); } internal static LiteException InvalidCommand(string command) { return new LiteException(121, "Command '{0}' is not a valid shell command.", command); } internal static LiteException AlreadyExistsCollectionName(string newName) { return new LiteException(122, "New collection name '{0}' already exists.", newName); } internal static LiteException AlreadyOpenDatafile(string filename) { return new LiteException(124, "Your datafile '{0}' is open in another process.", filename); } internal static LiteException InvalidDbRef(string path) { return new LiteException(113, "Invalid value for DbRef in path '{0}'. Value must be document like {{ $ref: \"?\", $id: ? }}", path); } internal static LiteException AlreadyExistsTransaction() { return new LiteException(126, "The current thread already contains an open transaction. Use the Commit/Rollback method to release the previous transaction."); } internal static LiteException CollectionLockerNotFound(string collection) { return new LiteException(126, "Collection locker '{0}' was not found inside dictionary.", collection); } internal static LiteException InvalidFormat(string field) { return new LiteException(200, "Invalid format: {0}", field); } internal static LiteException DocumentMaxDepth(int depth, Type type) { return new LiteException(201, "Document has more than {0} nested documents in '{1}'. Check for circular references (use DbRef).", depth, (type == null) ? "-" : type.Name); } internal static LiteException InvalidCtor(Type type, Exception inner) { return new LiteException(202, inner, "Failed to create instance for type '{0}' from assembly '{1}'. Checks if the class has a public constructor with no parameters.", type.FullName, type.AssemblyQualifiedName); } internal static LiteException UnexpectedToken(Token token, string expected = null) { long valueOrDefault = (token?.Position - (token?.Value?.Length).GetValueOrDefault()).GetValueOrDefault(); string arg = ((token != null && token.Type == TokenType.EOF) ? "[EOF]" : (token?.Value ?? "")); string arg2 = ((expected == null) ? "" : (" Expected `" + expected + "`.")); return new LiteException(203, $"Unexpected token `{arg}` in position {valueOrDefault}.{arg2}") { Position = valueOrDefault }; } internal static LiteException UnexpectedToken(string message, Token token) { long valueOrDefault = (token?.Position - (token?.Value?.Length).GetValueOrDefault()).GetValueOrDefault(); return new LiteException(203, message) { Position = valueOrDefault }; } internal static LiteException InvalidDataType(string field, BsonValue value) { return new LiteException(204, "Invalid BSON data type '{0}' on field '{1}'.", value.Type, field); } internal static LiteException PropertyReadWrite(PropertyInfo prop) { return new LiteException(209, "'{0}' property must have public getter and setter.", prop.Name); } internal static LiteException PropertyNotMapped(string name) { return new LiteException(206, "Property '{0}' was not mapped into BsonDocument.", name); } internal static LiteException InvalidTypedName(string type) { return new LiteException(207, "Type '{0}' not found in current domain (_type format is 'Type.FullName, AssemblyName').", type); } internal static LiteException InitialSizeCryptoNotSupported() { return new LiteException(210, "Initial Size option is not supported for encrypted datafiles."); } internal static LiteException InvalidInitialSize() { return new LiteException(211, "Initial Size must be a multiple of page size ({0} bytes).", 8192); } internal static LiteException EngineDisposed() { return new LiteException(137, "This engine instance already disposed."); } internal static LiteException InvalidNullCharInString() { return new LiteException(212, "Invalid null character (\\0) was found in the string"); } internal static LiteException InvalidPageType(PageType pageType, BasePage page) { StringBuilder stringBuilder = new StringBuilder($"Invalid {pageType} on {page.PageID}. "); stringBuilder.Append($"Full zero: {page.Buffer.All(0)}. "); stringBuilder.Append($"Page Type: {page.PageType}. "); stringBuilder.Append($"Prev/Next: {page.PrevPageID}/{page.NextPageID}. "); stringBuilder.Append($"UniqueID: {page.Buffer.UniqueID}. "); stringBuilder.Append($"ShareCounter: {page.Buffer.ShareCounter}. "); return new LiteException(0, stringBuilder.ToString()); } internal static LiteException InvalidFreeSpacePage(uint pageID, int freeBytes, int length) { return new LiteException(213, $"An operation that would corrupt page {pageID} was prevented. The operation required {length} free bytes, but the page had only {freeBytes} available."); } internal static LiteException DataTypeNotAssignable(string type1, string type2) { return new LiteException(214, "Data type " + type1 + " is not assignable from data type " + type2); } internal static LiteException FileNotEncrypted() { return new LiteException(216, "File is not encrypted."); } internal static LiteException InvalidPassword() { return new LiteException(217, "Invalid password."); } internal static LiteException IllegalDeserializationType(string typeName) { return new LiteException(218, "Illegal deserialization type: " + typeName); } internal static LiteException InvalidDatafileState(string message) { return new LiteException(999, message); } } internal class MimeTypeConverter { private static readonly IDictionary _mappings = new Dictionary(StringComparer.OrdinalIgnoreCase) { { ".323", "text/h323" }, { ".3g2", "video/3gpp2" }, { ".3gp", "video/3gpp" }, { ".3gp2", "video/3gpp2" }, { ".3gpp", "video/3gpp" }, { ".7z", "application/x-7z-compressed" }, { ".aa", "audio/audible" }, { ".AAC", "audio/aac" }, { ".aaf", "application/octet-stream" }, { ".aax", "audio/vnd.audible.aax" }, { ".ac3", "audio/ac3" }, { ".aca", "application/octet-stream" }, { ".accda", "application/msaccess.addin" }, { ".accdb", "application/msaccess" }, { ".accdc", "application/msaccess.cab" }, { ".accde", "application/msaccess" }, { ".accdr", "application/msaccess.runtime" }, { ".accdt", "application/msaccess" }, { ".accdw", "application/msaccess.webapplication" }, { ".accft", "application/msaccess.ftemplate" }, { ".acx", "application/internet-property-stream" }, { ".AddIn", "text/xml" }, { ".ade", "application/msaccess" }, { ".adobebridge", "application/x-bridge-url" }, { ".adp", "application/msaccess" }, { ".ADT", "audio/vnd.dlna.adts" }, { ".ADTS", "audio/aac" }, { ".afm", "application/octet-stream" }, { ".ai", "application/postscript" }, { ".aif", "audio/x-aiff" }, { ".aifc", "audio/aiff" }, { ".aiff", "audio/aiff" }, { ".air", "application/vnd.adobe.air-application-installer-package+zip" }, { ".amc", "application/x-mpeg" }, { ".application", "application/x-ms-application" }, { ".art", "image/x-jg" }, { ".asa", "application/xml" }, { ".asax", "application/xml" }, { ".ascx", "application/xml" }, { ".asd", "application/octet-stream" }, { ".asf", "video/x-ms-asf" }, { ".ashx", "application/xml" }, { ".asi", "application/octet-stream" }, { ".asm", "text/plain" }, { ".asmx", "application/xml" }, { ".aspx", "application/xml" }, { ".asr", "video/x-ms-asf" }, { ".asx", "video/x-ms-asf" }, { ".atom", "application/atom+xml" }, { ".au", "audio/basic" }, { ".avi", "video/x-msvideo" }, { ".axs", "application/olescript" }, { ".bas", "text/plain" }, { ".bcpio", "application/x-bcpio" }, { ".bin", "application/octet-stream" }, { ".bmp", "image/bmp" }, { ".c", "text/plain" }, { ".cab", "application/octet-stream" }, { ".caf", "audio/x-caf" }, { ".calx", "application/vnd.ms-office.calx" }, { ".cat", "application/vnd.ms-pki.seccat" }, { ".cc", "text/plain" }, { ".cd", "text/plain" }, { ".cdda", "audio/aiff" }, { ".cdf", "application/x-cdf" }, { ".cer", "application/x-x509-ca-cert" }, { ".chm", "application/octet-stream" }, { ".class", "application/x-java-applet" }, { ".clp", "application/x-msclip" }, { ".cmx", "image/x-cmx" }, { ".cnf", "text/plain" }, { ".cod", "image/cis-cod" }, { ".config", "application/xml" }, { ".contact", "text/x-ms-contact" }, { ".coverage", "application/xml" }, { ".cpio", "application/x-cpio" }, { ".cpp", "text/plain" }, { ".crd", "application/x-mscardfile" }, { ".crl", "application/pkix-crl" }, { ".crt", "application/x-x509-ca-cert" }, { ".cs", "text/plain" }, { ".csdproj", "text/plain" }, { ".csh", "application/x-csh" }, { ".csproj", "text/plain" }, { ".css", "text/css" }, { ".csv", "text/csv" }, { ".cur", "application/octet-stream" }, { ".cxx", "text/plain" }, { ".dat", "application/octet-stream" }, { ".datasource", "application/xml" }, { ".dbproj", "text/plain" }, { ".dcr", "application/x-director" }, { ".def", "text/plain" }, { ".deploy", "application/octet-stream" }, { ".der", "application/x-x509-ca-cert" }, { ".dgml", "application/xml" }, { ".dib", "image/bmp" }, { ".dif", "video/x-dv" }, { ".dir", "application/x-director" }, { ".disco", "text/xml" }, { ".dll", "application/x-msdownload" }, { ".dll.config", "text/xml" }, { ".dlm", "text/dlm" }, { ".doc", "application/msword" }, { ".docm", "application/vnd.ms-word.document.macroEnabled.12" }, { ".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document" }, { ".dot", "application/msword" }, { ".dotm", "application/vnd.ms-word.template.macroEnabled.12" }, { ".dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template" }, { ".dsp", "application/octet-stream" }, { ".dsw", "text/plain" }, { ".dtd", "text/xml" }, { ".dtsConfig", "text/xml" }, { ".dv", "video/x-dv" }, { ".dvi", "application/x-dvi" }, { ".dwf", "drawing/x-dwf" }, { ".dwp", "application/octet-stream" }, { ".dxr", "application/x-director" }, { ".eml", "message/rfc822" }, { ".emz", "application/octet-stream" }, { ".eot", "application/octet-stream" }, { ".eps", "application/postscript" }, { ".etl", "application/etl" }, { ".etx", "text/x-setext" }, { ".evy", "application/envoy" }, { ".exe", "application/octet-stream" }, { ".exe.config", "text/xml" }, { ".fdf", "application/vnd.fdf" }, { ".fif", "application/fractals" }, { ".filters", "Application/xml" }, { ".fla", "application/octet-stream" }, { ".flr", "x-world/x-vrml" }, { ".flv", "video/x-flv" }, { ".fsscript", "application/fsharp-script" }, { ".fsx", "application/fsharp-script" }, { ".generictest", "application/xml" }, { ".gif", "image/gif" }, { ".group", "text/x-ms-group" }, { ".gsm", "audio/x-gsm" }, { ".gtar", "application/x-gtar" }, { ".gz", "application/x-gzip" }, { ".h", "text/plain" }, { ".hdf", "application/x-hdf" }, { ".hdml", "text/x-hdml" }, { ".hhc", "application/x-oleobject" }, { ".hhk", "application/octet-stream" }, { ".hhp", "application/octet-stream" }, { ".hlp", "application/winhlp" }, { ".hpp", "text/plain" }, { ".hqx", "application/mac-binhex40" }, { ".hta", "application/hta" }, { ".htc", "text/x-component" }, { ".htm", "text/html" }, { ".html", "text/html" }, { ".htt", "text/webviewhtml" }, { ".hxa", "application/xml" }, { ".hxc", "application/xml" }, { ".hxd", "application/octet-stream" }, { ".hxe", "application/xml" }, { ".hxf", "application/xml" }, { ".hxh", "application/octet-stream" }, { ".hxi", "application/octet-stream" }, { ".hxk", "application/xml" }, { ".hxq", "application/octet-stream" }, { ".hxr", "application/octet-stream" }, { ".hxs", "application/octet-stream" }, { ".hxt", "text/html" }, { ".hxv", "application/xml" }, { ".hxw", "application/octet-stream" }, { ".hxx", "text/plain" }, { ".i", "text/plain" }, { ".ico", "image/x-icon" }, { ".ics", "application/octet-stream" }, { ".idl", "text/plain" }, { ".ief", "image/ief" }, { ".iii", "application/x-iphone" }, { ".inc", "text/plain" }, { ".inf", "application/octet-stream" }, { ".inl", "text/plain" }, { ".ins", "application/x-internet-signup" }, { ".ipa", "application/x-itunes-ipa" }, { ".ipg", "application/x-itunes-ipg" }, { ".ipproj", "text/plain" }, { ".ipsw", "application/x-itunes-ipsw" }, { ".iqy", "text/x-ms-iqy" }, { ".isp", "application/x-internet-signup" }, { ".ite", "application/x-itunes-ite" }, { ".itlp", "application/x-itunes-itlp" }, { ".itms", "application/x-itunes-itms" }, { ".itpc", "application/x-itunes-itpc" }, { ".IVF", "video/x-ivf" }, { ".jar", "application/java-archive" }, { ".java", "application/octet-stream" }, { ".jck", "application/liquidmotion" }, { ".jcz", "application/liquidmotion" }, { ".jfif", "image/pjpeg" }, { ".jnlp", "application/x-java-jnlp-file" }, { ".jpb", "application/octet-stream" }, { ".jpe", "image/jpeg" }, { ".jpeg", "image/jpeg" }, { ".jpg", "image/jpeg" }, { ".js", "application/x-javascript" }, { ".jsx", "text/jscript" }, { ".jsxbin", "text/plain" }, { ".latex", "application/x-latex" }, { ".library-ms", "application/windows-library+xml" }, { ".lit", "application/x-ms-reader" }, { ".loadtest", "application/xml" }, { ".lpk", "application/octet-stream" }, { ".lsf", "video/x-la-asf" }, { ".lst", "text/plain" }, { ".lsx", "video/x-la-asf" }, { ".lzh", "application/octet-stream" }, { ".m13", "application/x-msmediaview" }, { ".m14", "application/x-msmediaview" }, { ".m1v", "video/mpeg" }, { ".m2t", "video/vnd.dlna.mpeg-tts" }, { ".m2ts", "video/vnd.dlna.mpeg-tts" }, { ".m2v", "video/mpeg" }, { ".m3u", "audio/x-mpegurl" }, { ".m3u8", "audio/x-mpegurl" }, { ".m4a", "audio/m4a" }, { ".m4b", "audio/m4b" }, { ".m4p", "audio/m4p" }, { ".m4r", "audio/x-m4r" }, { ".m4v", "video/x-m4v" }, { ".mac", "image/x-macpaint" }, { ".mak", "text/plain" }, { ".man", "application/x-troff-man" }, { ".manifest", "application/x-ms-manifest" }, { ".map", "text/plain" }, { ".master", "application/xml" }, { ".mda", "application/msaccess" }, { ".mdb", "application/x-msaccess" }, { ".mde", "application/msaccess" }, { ".mdp", "application/octet-stream" }, { ".me", "application/x-troff-me" }, { ".mfp", "application/x-shockwave-flash" }, { ".mht", "message/rfc822" }, { ".mhtml", "message/rfc822" }, { ".mid", "audio/mid" }, { ".midi", "audio/mid" }, { ".mix", "application/octet-stream" }, { ".mk", "text/plain" }, { ".mmf", "application/x-smaf" }, { ".mno", "text/xml" }, { ".mny", "application/x-msmoney" }, { ".mod", "video/mpeg" }, { ".mov", "video/quicktime" }, { ".movie", "video/x-sgi-movie" }, { ".mp2", "video/mpeg" }, { ".mp2v", "video/mpeg" }, { ".mp3", "audio/mpeg" }, { ".mp4", "video/mp4" }, { ".mp4v", "video/mp4" }, { ".mpa", "video/mpeg" }, { ".mpe", "video/mpeg" }, { ".mpeg", "video/mpeg" }, { ".mpf", "application/vnd.ms-mediapackage" }, { ".mpg", "video/mpeg" }, { ".mpp", "application/vnd.ms-project" }, { ".mpv2", "video/mpeg" }, { ".mqv", "video/quicktime" }, { ".ms", "application/x-troff-ms" }, { ".msi", "application/octet-stream" }, { ".mso", "application/octet-stream" }, { ".mts", "video/vnd.dlna.mpeg-tts" }, { ".mtx", "application/xml" }, { ".mvb", "application/x-msmediaview" }, { ".mvc", "application/x-miva-compiled" }, { ".mxp", "application/x-mmxp" }, { ".nc", "application/x-netcdf" }, { ".nsc", "video/x-ms-asf" }, { ".nws", "message/rfc822" }, { ".ocx", "application/octet-stream" }, { ".oda", "application/oda" }, { ".odc", "text/x-ms-odc" }, { ".odh", "text/plain" }, { ".odl", "text/plain" }, { ".odp", "application/vnd.oasis.opendocument.presentation" }, { ".ods", "application/oleobject" }, { ".odt", "application/vnd.oasis.opendocument.text" }, { ".one", "application/onenote" }, { ".onea", "application/onenote" }, { ".onepkg", "application/onenote" }, { ".onetmp", "application/onenote" }, { ".onetoc", "application/onenote" }, { ".onetoc2", "application/onenote" }, { ".orderedtest", "application/xml" }, { ".osdx", "application/opensearchdescription+xml" }, { ".p10", "application/pkcs10" }, { ".p12", "application/x-pkcs12" }, { ".p7b", "application/x-pkcs7-certificates" }, { ".p7c", "application/pkcs7-mime" }, { ".p7m", "application/pkcs7-mime" }, { ".p7r", "application/x-pkcs7-certreqresp" }, { ".p7s", "application/pkcs7-signature" }, { ".pbm", "image/x-portable-bitmap" }, { ".pcast", "application/x-podcast" }, { ".pct", "image/pict" }, { ".pcx", "application/octet-stream" }, { ".pcz", "application/octet-stream" }, { ".pdf", "application/pdf" }, { ".pfb", "application/octet-stream" }, { ".pfm", "application/octet-stream" }, { ".pfx", "application/x-pkcs12" }, { ".pgm", "image/x-portable-graymap" }, { ".pic", "image/pict" }, { ".pict", "image/pict" }, { ".pkgdef", "text/plain" }, { ".pkgundef", "text/plain" }, { ".pko", "application/vnd.ms-pki.pko" }, { ".pls", "audio/scpls" }, { ".pma", "application/x-perfmon" }, { ".pmc", "application/x-perfmon" }, { ".pml", "application/x-perfmon" }, { ".pmr", "application/x-perfmon" }, { ".pmw", "application/x-perfmon" }, { ".png", "image/png" }, { ".pnm", "image/x-portable-anymap" }, { ".pnt", "image/x-macpaint" }, { ".pntg", "image/x-macpaint" }, { ".pnz", "image/png" }, { ".pot", "application/vnd.ms-powerpoint" }, { ".potm", "application/vnd.ms-powerpoint.template.macroEnabled.12" }, { ".potx", "application/vnd.openxmlformats-officedocument.presentationml.template" }, { ".ppa", "application/vnd.ms-powerpoint" }, { ".ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12" }, { ".ppm", "image/x-portable-pixmap" }, { ".pps", "application/vnd.ms-powerpoint" }, { ".ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12" }, { ".ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow" }, { ".ppt", "application/vnd.ms-powerpoint" }, { ".pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12" }, { ".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" }, { ".prf", "application/pics-rules" }, { ".prm", "application/octet-stream" }, { ".prx", "application/octet-stream" }, { ".ps", "application/postscript" }, { ".psc1", "application/PowerShell" }, { ".psd", "application/octet-stream" }, { ".psess", "application/xml" }, { ".psm", "application/octet-stream" }, { ".psp", "application/octet-stream" }, { ".pub", "application/x-mspublisher" }, { ".pwz", "application/vnd.ms-powerpoint" }, { ".qht", "text/x-html-insertion" }, { ".qhtm", "text/x-html-insertion" }, { ".qt", "video/quicktime" }, { ".qti", "image/x-quicktime" }, { ".qtif", "image/x-quicktime" }, { ".qtl", "application/x-quicktimeplayer" }, { ".qxd", "application/octet-stream" }, { ".ra", "audio/x-pn-realaudio" }, { ".ram", "audio/x-pn-realaudio" }, { ".rar", "application/octet-stream" }, { ".ras", "image/x-cmu-raster" }, { ".rat", "application/rat-file" }, { ".rc", "text/plain" }, { ".rc2", "text/plain" }, { ".rct", "text/plain" }, { ".rdlc", "application/xml" }, { ".resx", "application/xml" }, { ".rf", "image/vnd.rn-realflash" }, { ".rgb", "image/x-rgb" }, { ".rgs", "text/plain" }, { ".rm", "application/vnd.rn-realmedia" }, { ".rmi", "audio/mid" }, { ".rmp", "application/vnd.rn-rn_music_package" }, { ".roff", "application/x-troff" }, { ".rpm", "audio/x-pn-realaudio-plugin" }, { ".rqy", "text/x-ms-rqy" }, { ".rtf", "application/rtf" }, { ".rtx", "text/richtext" }, { ".ruleset", "application/xml" }, { ".s", "text/plain" }, { ".safariextz", "application/x-safari-safariextz" }, { ".scd", "application/x-msschedule" }, { ".sct", "text/scriptlet" }, { ".sd2", "audio/x-sd2" }, { ".sdp", "application/sdp" }, { ".sea", "application/octet-stream" }, { ".searchConnector-ms", "application/windows-search-connector+xml" }, { ".setpay", "application/set-payment-initiation" }, { ".setreg", "application/set-registration-initiation" }, { ".settings", "application/xml" }, { ".sgimb", "application/x-sgimb" }, { ".sgml", "text/sgml" }, { ".sh", "application/x-sh" }, { ".shar", "application/x-shar" }, { ".shtml", "text/html" }, { ".sit", "application/x-stuffit" }, { ".sitemap", "application/xml" }, { ".skin", "application/xml" }, { ".sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12" }, { ".sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide" }, { ".slk", "application/vnd.ms-excel" }, { ".sln", "text/plain" }, { ".slupkg-ms", "application/x-ms-license" }, { ".smd", "audio/x-smd" }, { ".smi", "application/octet-stream" }, { ".smx", "audio/x-smd" }, { ".smz", "audio/x-smd" }, { ".snd", "audio/basic" }, { ".snippet", "application/xml" }, { ".snp", "application/octet-stream" }, { ".sol", "text/plain" }, { ".sor", "text/plain" }, { ".spc", "application/x-pkcs7-certificates" }, { ".spl", "application/futuresplash" }, { ".src", "application/x-wais-source" }, { ".srf", "text/plain" }, { ".SSISDeploymentManifest", "text/xml" }, { ".ssm", "application/streamingmedia" }, { ".sst", "application/vnd.ms-pki.certstore" }, { ".stl", "application/vnd.ms-pki.stl" }, { ".sv4cpio", "application/x-sv4cpio" }, { ".sv4crc", "application/x-sv4crc" }, { ".svc", "application/xml" }, { ".swf", "application/x-shockwave-flash" }, { ".t", "application/x-troff" }, { ".tar", "application/x-tar" }, { ".tcl", "application/x-tcl" }, { ".testrunconfig", "application/xml" }, { ".testsettings", "application/xml" }, { ".tex", "application/x-tex" }, { ".texi", "application/x-texinfo" }, { ".texinfo", "application/x-texinfo" }, { ".tgz", "application/x-compressed" }, { ".thmx", "application/vnd.ms-officetheme" }, { ".thn", "application/octet-stream" }, { ".tif", "image/tiff" }, { ".tiff", "image/tiff" }, { ".tlh", "text/plain" }, { ".tli", "text/plain" }, { ".toc", "application/octet-stream" }, { ".tr", "application/x-troff" }, { ".trm", "application/x-msterminal" }, { ".trx", "application/xml" }, { ".ts", "video/vnd.dlna.mpeg-tts" }, { ".tsv", "text/tab-separated-values" }, { ".ttf", "application/octet-stream" }, { ".tts", "video/vnd.dlna.mpeg-tts" }, { ".txt", "text/plain" }, { ".u32", "application/octet-stream" }, { ".uls", "text/iuls" }, { ".user", "text/plain" }, { ".ustar", "application/x-ustar" }, { ".vb", "text/plain" }, { ".vbdproj", "text/plain" }, { ".vbk", "video/mpeg" }, { ".vbproj", "text/plain" }, { ".vbs", "text/vbscript" }, { ".vcf", "text/x-vcard" }, { ".vcproj", "Application/xml" }, { ".vcs", "text/plain" }, { ".vcxproj", "Application/xml" }, { ".vddproj", "text/plain" }, { ".vdp", "text/plain" }, { ".vdproj", "text/plain" }, { ".vdx", "application/vnd.ms-visio.viewer" }, { ".vml", "text/xml" }, { ".vscontent", "application/xml" }, { ".vsct", "text/xml" }, { ".vsd", "application/vnd.visio" }, { ".vsi", "application/ms-vsi" }, { ".vsix", "application/vsix" }, { ".vsixlangpack", "text/xml" }, { ".vsixmanifest", "text/xml" }, { ".vsmdi", "application/xml" }, { ".vspscc", "text/plain" }, { ".vss", "application/vnd.visio" }, { ".vsscc", "text/plain" }, { ".vssettings", "text/xml" }, { ".vssscc", "text/plain" }, { ".vst", "application/vnd.visio" }, { ".vstemplate", "text/xml" }, { ".vsto", "application/x-ms-vsto" }, { ".vsw", "application/vnd.visio" }, { ".vsx", "application/vnd.visio" }, { ".vtx", "application/vnd.visio" }, { ".wav", "audio/wav" }, { ".wave", "audio/wav" }, { ".wax", "audio/x-ms-wax" }, { ".wbk", "application/msword" }, { ".wbmp", "image/vnd.wap.wbmp" }, { ".wcm", "application/vnd.ms-works" }, { ".wdb", "application/vnd.ms-works" }, { ".wdp", "image/vnd.ms-photo" }, { ".webarchive", "application/x-safari-webarchive" }, { ".webp", "image/webp" }, { ".webtest", "application/xml" }, { ".wiq", "application/xml" }, { ".wiz", "application/msword" }, { ".wks", "application/vnd.ms-works" }, { ".WLMP", "application/wlmoviemaker" }, { ".wlpginstall", "application/x-wlpg-detect" }, { ".wlpginstall3", "application/x-wlpg3-detect" }, { ".wm", "video/x-ms-wm" }, { ".wma", "audio/x-ms-wma" }, { ".wmd", "application/x-ms-wmd" }, { ".wmf", "application/x-msmetafile" }, { ".wml", "text/vnd.wap.wml" }, { ".wmlc", "application/vnd.wap.wmlc" }, { ".wmls", "text/vnd.wap.wmlscript" }, { ".wmlsc", "application/vnd.wap.wmlscriptc" }, { ".wmp", "video/x-ms-wmp" }, { ".wmv", "video/x-ms-wmv" }, { ".wmx", "video/x-ms-wmx" }, { ".wmz", "application/x-ms-wmz" }, { ".wpl", "application/vnd.ms-wpl" }, { ".wps", "application/vnd.ms-works" }, { ".wri", "application/x-mswrite" }, { ".wrl", "x-world/x-vrml" }, { ".wrz", "x-world/x-vrml" }, { ".wsc", "text/scriptlet" }, { ".wsdl", "text/xml" }, { ".wvx", "video/x-ms-wvx" }, { ".x", "application/directx" }, { ".xaf", "x-world/x-vrml" }, { ".xaml", "application/xaml+xml" }, { ".xap", "application/x-silverlight-app" }, { ".xbap", "application/x-ms-xbap" }, { ".xbm", "image/x-xbitmap" }, { ".xdr", "text/plain" }, { ".xht", "application/xhtml+xml" }, { ".xhtml", "application/xhtml+xml" }, { ".xla", "application/vnd.ms-excel" }, { ".xlam", "application/vnd.ms-excel.addin.macroEnabled.12" }, { ".xlc", "application/vnd.ms-excel" }, { ".xld", "application/vnd.ms-excel" }, { ".xlk", "application/vnd.ms-excel" }, { ".xll", "application/vnd.ms-excel" }, { ".xlm", "application/vnd.ms-excel" }, { ".xls", "application/vnd.ms-excel" }, { ".xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12" }, { ".xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12" }, { ".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" }, { ".xlt", "application/vnd.ms-excel" }, { ".xltm", "application/vnd.ms-excel.template.macroEnabled.12" }, { ".xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template" }, { ".xlw", "application/vnd.ms-excel" }, { ".xml", "text/xml" }, { ".xmta", "application/xml" }, { ".xof", "x-world/x-vrml" }, { ".XOML", "text/plain" }, { ".xpm", "image/x-xpixmap" }, { ".xps", "application/vnd.ms-xpsdocument" }, { ".xrm-ms", "text/xml" }, { ".xsc", "application/xml" }, { ".xsd", "text/xml" }, { ".xsf", "text/xml" }, { ".xsl", "text/xml" }, { ".xslt", "text/xml" }, { ".xsn", "application/octet-stream" }, { ".xss", "application/xml" }, { ".xtp", "application/octet-stream" }, { ".xwd", "image/x-xwindowdump" }, { ".z", "application/x-compress" }, { ".zip", "application/x-zip-compressed" } }; public static string GetMimeType(string filename) { if (string.IsNullOrEmpty(filename)) { throw new ArgumentNullException("filename"); } string text = Path.GetExtension(filename); if (!text.StartsWith(".")) { text = "." + text; } if (!_mappings.TryGetValue(text, out var value)) { return "application/octet-stream"; } return value; } } internal static class BucketHelper { public const int BucketCount = 17; private static readonly int[] _bucketSize; static BucketHelper() { _bucketSize = new int[17]; for (int i = 0; i < 17; i++) { _bucketSize[i] = GetMaxSizeForBucket(i); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int GetBucketIndex(int bufferSize) { for (int i = 0; i < _bucketSize.Length; i++) { if (_bucketSize[i] >= bufferSize) { return i; } } return -1; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int GetMaxSizeForBucket(int binIndex) { return 16 << binIndex; } } internal static class Randomizer { private static readonly Random _random = new Random(0); public static int Next() { lock (_random) { return _random.Next(); } } public static int Next(int minValue, int maxValue) { lock (_random) { return _random.Next(minValue, maxValue); } } } internal struct Result where T : class { public T Value; public Exception Exception; public bool Ok => Exception == null; public bool Fail => Exception != null; public T GetValue() { if (!Ok) { throw Exception; } return Value; } public Result(T value, Exception ex = null) { Value = value; Exception = ex; } public static implicit operator T(Result value) { return value.Value; } public static implicit operator Result(T value) { return new Result(value); } } internal enum TokenType { OpenBrace, CloseBrace, OpenBracket, CloseBracket, OpenParenthesis, CloseParenthesis, Comma, Colon, SemiColon, At, Hashtag, Til, Period, Ampersand, Dollar, Exclamation, NotEquals, Equals, Greater, GreaterOrEquals, Less, LessOrEquals, Minus, Plus, Asterisk, Slash, Backslash, Percent, String, Int, Double, Whitespace, Word, EOF, Unknown } internal class Token { private static readonly HashSet _keywords = new HashSet(StringComparer.OrdinalIgnoreCase) { "BETWEEN", "LIKE", "IN", "AND", "OR" }; public TokenType Type { get; private set; } public string Value { get; private set; } public long Position { get; private set; } public bool IsOperand { get { switch (Type) { case TokenType.NotEquals: case TokenType.Equals: case TokenType.Greater: case TokenType.GreaterOrEquals: case TokenType.Less: case TokenType.LessOrEquals: case TokenType.Minus: case TokenType.Plus: case TokenType.Asterisk: case TokenType.Slash: case TokenType.Percent: return true; case TokenType.Word: return _keywords.Contains(Value); default: return false; } } } public Token(TokenType tokenType, string value, long position) { Position = position; Value = value; Type = tokenType; } public Token Expect(TokenType type) { if (Type != type) { throw LiteException.UnexpectedToken(this); } return this; } public Token Expect(TokenType type1, TokenType type2) { if (Type != type1 && Type != type2) { throw LiteException.UnexpectedToken(this); } return this; } public Token Expect(TokenType type1, TokenType type2, TokenType type3) { if (Type != type1 && Type != type2 && Type != type3) { throw LiteException.UnexpectedToken(this); } return this; } public Token Expect(string value, bool ignoreCase = true) { if (!Is(value, ignoreCase)) { throw LiteException.UnexpectedToken(this, value); } return this; } public bool Is(string value, bool ignoreCase = true) { if (Type == TokenType.Word) { return value.Equals(Value, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); } return false; } public override string ToString() { return Value + " (" + Type.ToString() + ")"; } } internal class Tokenizer { private readonly TextReader _reader; private char _char; private Token _ahead; private bool _eof; public bool EOF { get { if (_eof) { return _ahead == null; } return false; } } public long Position { get; private set; } public Token Current { get; private set; } public bool CheckEOF() { if (_eof) { throw LiteException.UnexpectedToken(Current); } return false; } public Tokenizer(string source) : this(new StringReader(source)) { } public Tokenizer(TextReader reader) { _reader = reader; Position = 0L; ReadChar(); } public static bool IsWordChar(char c, bool first) { if (first) { if (!char.IsLetter(c) && c != '_') { return c == '$'; } return true; } if (!char.IsLetterOrDigit(c) && c != '_') { return c == '$'; } return true; } private char ReadChar() { if (_eof) { return '\0'; } int num = _reader.Read(); Position++; if (num == -1) { _char = '\0'; _eof = true; } else { _char = (char)num; } return _char; } public Token LookAhead(bool eatWhitespace = true) { if (_ahead != null) { if (eatWhitespace && _ahead.Type == TokenType.Whitespace) { _ahead = ReadNext(eatWhitespace); } return _ahead; } return _ahead = ReadNext(eatWhitespace); } public Token ReadToken(bool eatWhitespace = true) { if (_ahead == null) { return Current = ReadNext(eatWhitespace); } if (eatWhitespace && _ahead.Type == TokenType.Whitespace) { _ahead = ReadNext(eatWhitespace); } Current = _ahead; _ahead = null; return Current; } private Token ReadNext(bool eatWhitespace) { if (eatWhitespace) { EatWhitespace(); } if (_eof) { return new Token(TokenType.EOF, null, Position); } Token token = null; switch (_char) { case '{': token = new Token(TokenType.OpenBrace, "{", Position); ReadChar(); break; case '}': token = new Token(TokenType.CloseBrace, "}", Position); ReadChar(); break; case '[': token = new Token(TokenType.OpenBracket, "[", Position); ReadChar(); break; case ']': token = new Token(TokenType.CloseBracket, "]", Position); ReadChar(); break; case '(': token = new Token(TokenType.OpenParenthesis, "(", Position); ReadChar(); break; case ')': token = new Token(TokenType.CloseParenthesis, ")", Position); ReadChar(); break; case ',': token = new Token(TokenType.Comma, ",", Position); ReadChar(); break; case ':': token = new Token(TokenType.Colon, ":", Position); ReadChar(); break; case ';': token = new Token(TokenType.SemiColon, ";", Position); ReadChar(); break; case '@': token = new Token(TokenType.At, "@", Position); ReadChar(); break; case '#': token = new Token(TokenType.Hashtag, "#", Position); ReadChar(); break; case '~': token = new Token(TokenType.Til, "~", Position); ReadChar(); break; case '.': token = new Token(TokenType.Period, ".", Position); ReadChar(); break; case '&': token = new Token(TokenType.Ampersand, "&", Position); ReadChar(); break; case '$': ReadChar(); token = ((!IsWordChar(_char, first: true)) ? new Token(TokenType.Dollar, "$", Position) : new Token(TokenType.Word, "$" + ReadWord(), Position)); break; case '!': ReadChar(); if (_char == '=') { token = new Token(TokenType.NotEquals, "!=", Position); ReadChar(); } else { token = new Token(TokenType.Exclamation, "!", Position); } break; case '=': token = new Token(TokenType.Equals, "=", Position); ReadChar(); break; case '>': ReadChar(); if (_char == '=') { token = new Token(TokenType.GreaterOrEquals, ">=", Position); ReadChar(); } else { token = new Token(TokenType.Greater, ">", Position); } break; case '<': ReadChar(); if (_char == '=') { token = new Token(TokenType.LessOrEquals, "<=", Position); ReadChar(); } else { token = new Token(TokenType.Less, "<", Position); } break; case '-': ReadChar(); if (_char == '-') { ReadLine(); token = ReadNext(eatWhitespace); } else { token = new Token(TokenType.Minus, "-", Position); } break; case '+': token = new Token(TokenType.Plus, "+", Position); ReadChar(); break; case '*': token = new Token(TokenType.Asterisk, "*", Position); ReadChar(); break; case '/': token = new Token(TokenType.Slash, "/", Position); ReadChar(); break; case '\\': token = new Token(TokenType.Backslash, "\\", Position); ReadChar(); break; case '%': token = new Token(TokenType.Percent, "%", Position); ReadChar(); break; case '"': case '\'': token = new Token(TokenType.String, ReadString(_char), Position); break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { bool dbl = false; string value = ReadNumber(ref dbl); token = new Token(dbl ? TokenType.Double : TokenType.Int, value, Position); break; } case '\t': case '\n': case '\r': case ' ': { StringBuilder stringBuilder = new StringBuilder(); while (char.IsWhiteSpace(_char) && !_eof) { stringBuilder.Append(_char); ReadChar(); } token = new Token(TokenType.Whitespace, stringBuilder.ToString(), Position); break; } default: if (IsWordChar(_char, first: true)) { token = new Token(TokenType.Word, ReadWord(), Position); } else { ReadChar(); } break; } return token ?? new Token(TokenType.Unknown, _char.ToString(), Position); } private void EatWhitespace() { while (char.IsWhiteSpace(_char) && !_eof) { ReadChar(); } } private string ReadWord() { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.Append(_char); ReadChar(); while (!_eof && IsWordChar(_char, first: false)) { stringBuilder.Append(_char); ReadChar(); } return stringBuilder.ToString(); } private string ReadNumber(ref bool dbl) { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.Append(_char); bool flag = true; bool flag2 = true; bool flag3 = false; ReadChar(); while (!_eof && (char.IsDigit(_char) || _char == '+' || _char == '-' || _char == '.' || _char == 'e' || _char == 'E')) { if (_char == '.') { if (!flag) { break; } dbl = true; flag = false; } else if (_char == 'e' || _char == 'E') { if (!flag2) { break; } flag2 = false; flag3 = true; dbl = true; } else if (_char == '-' || _char == '+') { if (!flag3) { break; } flag3 = false; } stringBuilder.Append(_char); ReadChar(); } return stringBuilder.ToString(); } private string ReadString(char quote) { StringBuilder stringBuilder = new StringBuilder(); ReadChar(); while (_char != quote && !_eof) { if (_char == '\\') { ReadChar(); if (_char == quote) { stringBuilder.Append(quote); } switch (_char) { case '\\': stringBuilder.Append('\\'); break; case '/': stringBuilder.Append('/'); break; case 'b': stringBuilder.Append('\b'); break; case 'f': stringBuilder.Append('\f'); break; case 'n': stringBuilder.Append('\n'); break; case 'r': stringBuilder.Append('\r'); break; case 't': stringBuilder.Append('\t'); break; case 'u': { uint num = ParseUnicode(ReadChar(), ReadChar(), ReadChar(), ReadChar()); stringBuilder.Append((char)num); break; } } } else { stringBuilder.Append(_char); } ReadChar(); } ReadChar(); return stringBuilder.ToString(); } private void ReadLine() { while (_char != '\n' && !_eof) { ReadChar(); } if (_char == '\n') { ReadChar(); } } public static uint ParseUnicode(char c1, char c2, char c3, char c4) { uint num = ParseSingleChar(c1, 4096u); uint num2 = ParseSingleChar(c2, 256u); uint num3 = ParseSingleChar(c3, 16u); uint num4 = ParseSingleChar(c4, 1u); return num + num2 + num3 + num4; } public static uint ParseSingleChar(char c1, uint multiplier) { uint result = 0u; if (c1 >= '0' && c1 <= '9') { result = (uint)(c1 - 48) * multiplier; } else if (c1 >= 'A' && c1 <= 'F') { result = (uint)(c1 - 65 + 10) * multiplier; } else if (c1 >= 'a' && c1 <= 'f') { result = (uint)(c1 - 97 + 10) * multiplier; } return result; } public override string ToString() { return Current?.ToString() + " [ahead: " + _ahead?.ToString() + "] - position: " + Position; } } } namespace LiteDB.Utils { internal class TryCatch { public readonly List Exceptions = new List(); public bool InvalidDatafileState => Exceptions.Any((Exception ex) => ex is LiteException ex2 && ex2.ErrorCode == 999); public TryCatch() { } public TryCatch(Exception initial) { Exceptions.Add(initial); } [DebuggerHidden] public void Catch(Action action) { try { action(); } catch (Exception item) { Exceptions.Add(item); } } } } namespace LiteDB.Utils.Extensions { internal static class EnumerableExtensions { public static IEnumerable OnDispose(this IEnumerable source, Action onDispose) { try { foreach (T item in source) { yield return item; } } finally { onDispose(); } } } public static class StopWatchExtensions { private class DisposableAction : IDisposable { private readonly Action _action; public DisposableAction(Action action) { _action = action; } public void Dispose() { _action(); } } public static IDisposable StartDisposable(this Stopwatch stopwatch) { stopwatch.Start(); return new DisposableAction(stopwatch.Stop); } } } namespace LiteDB.Engine { internal class DiskReader : IDisposable { private readonly EngineState _state; private readonly MemoryCache _cache; private readonly StreamPool _dataPool; private readonly StreamPool _logPool; private readonly Lazy _dataStream; private readonly Lazy _logStream; public DiskReader(EngineState state, MemoryCache cache, StreamPool dataPool, StreamPool logPool) { _state = state; _cache = cache; _dataPool = dataPool; _logPool = logPool; _dataStream = new Lazy(() => _dataPool.Rent()); _logStream = new Lazy(() => _logPool.Rent()); } public PageBuffer ReadPage(long position, bool writable, FileOrigin origin) { Constants.ENSURE(position % 8192 == 0, "invalid page position"); Stream stream = ((origin == FileOrigin.Data) ? _dataStream.Value : _logStream.Value); if (!writable) { return _cache.GetReadablePage(position, origin, delegate(long pos, BufferSlice buf) { ReadStream(stream, pos, buf); }); } return _cache.GetWritablePage(position, origin, delegate(long pos, BufferSlice buf) { ReadStream(stream, pos, buf); }); } private void ReadStream(Stream stream, long position, BufferSlice buffer) { stream.Position = position; stream.Read(buffer.Array, buffer.Offset, buffer.Count); } public PageBuffer NewPage() { return _cache.NewPage(); } public void Dispose() { if (_dataStream.IsValueCreated) { _dataPool.Return(_dataStream.Value); } if (_logStream.IsValueCreated) { _logPool.Return(_logStream.Value); } } } internal class DiskService : IDisposable { private readonly MemoryCache _cache; private readonly EngineState _state; private IStreamFactory _dataFactory; private readonly IStreamFactory _logFactory; private StreamPool _dataPool; private readonly StreamPool _logPool; private readonly Lazy _writer; private long _dataLength; private long _logLength; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public MemoryCache Cache => _cache; public uint MAX_ITEMS_COUNT => (uint)((int)((_dataLength + _logLength) / 8192 + 10) * 255); public DiskService(EngineSettings settings, EngineState state, int[] memorySegmentSizes) { _cache = new MemoryCache(memorySegmentSizes); _state = state; _dataFactory = settings.CreateDataFactory(); _logFactory = settings.CreateLogFactory(); _dataPool = new StreamPool(_dataFactory, appendOnly: false); _logPool = new StreamPool(_logFactory, appendOnly: true); _writer = _logPool.Writer; if (_dataFactory.GetLength() == 0) { Initialize(_dataPool.Writer.Value, settings.Collation, settings.InitialSize); } if (!settings.ReadOnly) { _ = _dataPool.Writer.Value.CanRead; } _dataLength = _dataFactory.GetLength() - 8192; if (_logFactory.Exists()) { _logLength = _logFactory.GetLength() - 8192; } else { _logLength = -8192L; } } private void Initialize(Stream stream, Collation collation, long initialSize) { PageBuffer pageBuffer = new PageBuffer(new byte[8192], 0, 0); HeaderPage headerPage = new HeaderPage(pageBuffer, 0u); headerPage.Pragmas.Set("COLLATION", (collation ?? Collation.Default).ToString(), validate: false); headerPage.UpdateBuffer(); stream.Write(pageBuffer.Array, pageBuffer.Offset, 8192); if (initialSize > 0) { if (stream is AesStream) { throw LiteException.InitialSizeCryptoNotSupported(); } if (initialSize % 8192 != 0L) { throw LiteException.InvalidInitialSize(); } stream.SetLength(initialSize); } stream.FlushToDisk(); } public DiskReader GetReader() { return new DiskReader(_state, _cache, _dataPool, _logPool); } public void DiscardDirtyPages(IEnumerable pages) { foreach (PageBuffer page in pages) { _cache.DiscardPage(page); } } public void DiscardCleanPages(IEnumerable pages) { foreach (PageBuffer page in pages) { if (!_cache.TryMoveToReadable(page)) { _cache.DiscardPage(page); } } } public PageBuffer NewPage() { return _cache.NewPage(); } public int WriteLogDisk(IEnumerable pages) { int num = 0; Stream value = _writer.Value; lock (value) { foreach (PageBuffer page in pages) { Constants.ENSURE(page.ShareCounter == Constants.BUFFER_WRITABLE, "to enqueue page, page must be writable"); page.Position = Interlocked.Add(ref _logLength, 8192L); page.Origin = FileOrigin.Log; _cache.MoveToReadable(page); value.Position = page.Position; value.Write(page.Array, page.Offset, 8192); page.Release(); num++; } return num; } } public long GetFileLength(FileOrigin origin) { if (origin == FileOrigin.Log) { return _logLength + 8192; } return _dataLength + 8192; } internal void MarkAsInvalidState() { FileHelper.TryExec(60, delegate { using Stream stream = _dataFactory.GetStream(canWrite: true, sequencial: true); byte[] array = _bufferPool.Rent(8192); stream.Read(array, 0, 8192); array[191] = 1; stream.Position = 0L; stream.Write(array, 0, 8192); _bufferPool.Return(array, clearArray: true); }); } public IEnumerable ReadFull(FileOrigin origin) { byte[] buffer = new byte[8192]; StreamPool pool = ((origin == FileOrigin.Log) ? _logPool : _dataPool); Stream stream = pool.Rent(); try { long length = GetFileLength(origin); stream.Position = 0L; while (stream.Position < length) { long position = stream.Position; int num = stream.Read(buffer, 0, 8192); Constants.ENSURE(num == 8192, "ReadFull must read PAGE_SIZE bytes [{0}]", num); yield return new PageBuffer(buffer, 0, 0) { Position = position, Origin = origin, ShareCounter = 0 }; } } finally { pool.Return(stream); } } public void WriteDataDisk(IEnumerable pages) { Stream value = _dataPool.Writer.Value; foreach (PageBuffer page in pages) { Constants.ENSURE(page.ShareCounter == 0, "this page can't be shared to use sync operation - do not use cached pages"); _dataLength = Math.Max(_dataLength, page.Position); value.Position = page.Position; value.Write(page.Array, page.Offset, 8192); } value.FlushToDisk(); } public void SetLength(long length, FileOrigin origin) { Lazy obj = ((origin == FileOrigin.Log) ? _logPool.Writer : _dataPool.Writer); if (origin == FileOrigin.Log) { Interlocked.Exchange(ref _logLength, length - 8192); } else { Interlocked.Exchange(ref _dataLength, length - 8192); } obj.Value.SetLength(length); } public string GetName(FileOrigin origin) { if (origin != FileOrigin.Data) { return _logFactory.Name; } return _dataFactory.Name; } public void Dispose() { bool num = _logFactory.Exists() && _logPool.Writer.Value.Length == 0; _dataPool.Dispose(); _logPool.Dispose(); if (num) { _logFactory.Delete(); } _cache.Dispose(); } } internal class MemoryCache : IDisposable { private readonly ConcurrentQueue _free = new ConcurrentQueue(); private readonly ConcurrentDictionary _readable = new ConcurrentDictionary(); private int _extends; private readonly int[] _segmentSizes; public int PagesInUse => _readable.Values.Where((PageBuffer x) => x.ShareCounter != 0).Count(); public int FreePages => _free.Count; public int ExtendSegments => _extends; public int ExtendPages => (from x in Enumerable.Range(0, _extends) select _segmentSizes[Math.Min(_segmentSizes.Length - 1, x)]).Sum(); public int WritablePages => ExtendPages - _free.Count - _readable.Count; public MemoryCache(int[] memorySegmentSizes) { _segmentSizes = memorySegmentSizes; Extend(); } public PageBuffer GetReadablePage(long position, FileOrigin origin, Action factory) { long readableKey = GetReadableKey(position, origin); PageBuffer orAdd = _readable.GetOrAdd(readableKey, delegate { PageBuffer freePage = GetFreePage(); freePage.Position = position; freePage.Origin = origin; factory(position, freePage); return freePage; }); Interlocked.Exchange(ref orAdd.Timestamp, DateTime.UtcNow.Ticks); Interlocked.Increment(ref orAdd.ShareCounter); return orAdd; } private long GetReadableKey(long position, FileOrigin origin) { Constants.ENSURE(origin != FileOrigin.None, "file origin must be defined"); if (origin == FileOrigin.Data) { return position; } if (position == 0L) { return long.MinValue; } return -position; } public PageBuffer GetWritablePage(long position, FileOrigin origin, Action factory) { long readableKey = GetReadableKey(position, origin); PageBuffer pageBuffer = NewPage(position, origin); if (_readable.TryGetValue(readableKey, out var value)) { Buffer.BlockCopy(value.Array, value.Offset, pageBuffer.Array, pageBuffer.Offset, 8192); } else { factory(position, pageBuffer); } return pageBuffer; } public PageBuffer NewPage() { return NewPage(long.MaxValue, FileOrigin.None); } private PageBuffer NewPage(long position, FileOrigin origin) { PageBuffer freePage = GetFreePage(); freePage.Position = position; freePage.ShareCounter = Constants.BUFFER_WRITABLE; if (freePage.Timestamp > 0) { freePage.Clear(); } freePage.Origin = origin; freePage.Timestamp = DateTime.UtcNow.Ticks; return freePage; } public bool TryMoveToReadable(PageBuffer page) { Constants.ENSURE(page.Position != long.MaxValue, "page must have a position"); Constants.ENSURE(page.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable"); Constants.ENSURE(page.Origin != FileOrigin.None, "page must have origin defined"); long readableKey = GetReadableKey(page.Position, page.Origin); page.ShareCounter = 0; bool num = _readable.TryAdd(readableKey, page); if (!num) { page.ShareCounter = Constants.BUFFER_WRITABLE; } return num; } public PageBuffer MoveToReadable(PageBuffer page) { Constants.ENSURE(page.Position != long.MaxValue, "page must have position to be readable"); Constants.ENSURE(page.Origin != FileOrigin.None, "page should be a source before move to readable"); Constants.ENSURE(page.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable before move to readable dict"); long readableKey = GetReadableKey(page.Position, page.Origin); bool added = true; page.ShareCounter = 1; PageBuffer result = _readable.AddOrUpdate(readableKey, page, delegate(long newKey, PageBuffer current) { Constants.ENSURE(current.ShareCounter == 0, "user must ensure this page is not in use when marked as read only"); Constants.ENSURE(current.Origin == page.Origin, "origin must be same"); current.ShareCounter = 1; Buffer.BlockCopy(page.Array, page.Offset, current.Array, current.Offset, 8192); added = false; page.ShareCounter = Constants.BUFFER_WRITABLE; return current; }); if (!added) { DiscardPage(page); } return result; } public void DiscardPage(PageBuffer page) { Constants.ENSURE(page.ShareCounter == Constants.BUFFER_WRITABLE, "discarded page must be writable"); page.ShareCounter = 0; page.Position = long.MaxValue; page.Origin = FileOrigin.None; _free.Enqueue(page); } private PageBuffer GetFreePage() { if (_free.TryDequeue(out var result)) { Constants.ENSURE(result.Position == long.MaxValue, "pages in memory store must have no position defined"); Constants.ENSURE(result.ShareCounter == 0, "pages in memory store must be non-shared"); Constants.ENSURE(result.Origin == FileOrigin.None, "page in memory must have no page origin"); return result; } lock (_free) { if (_free.Count > 0) { return GetFreePage(); } Extend(); } return GetFreePage(); } private void Extend() { int num = _readable.Values.Count((PageBuffer x) => x.ShareCounter == 0); int num2 = _segmentSizes[Math.Min(_segmentSizes.Length - 1, _extends)]; if (num > num2) { long[] array = (from x in _readable where x.Value.ShareCounter == 0 orderby x.Value.Timestamp select x.Key).Take(num2).ToArray(); foreach (long key in array) { Constants.ENSURE(_readable.TryRemove(key, out var value), "page should be in readable list before moving to free list"); if (value.ShareCounter > 0) { if (!_readable.TryAdd(key, value)) { throw new LiteException(0, "MemoryCache: removed in-use memory page. This situation has no way to fix (yet). Throwing exception to avoid database corruption. No other thread can read/write from database now."); } } else { Constants.ENSURE(value.ShareCounter == 0, "page should not be in use by anyone"); value.Position = long.MaxValue; value.Origin = FileOrigin.None; _free.Enqueue(value); } } } else { byte[] buffer = new byte[8192 * num2]; int num3 = ExtendPages + 1; for (int j = 0; j < num2; j++) { _free.Enqueue(new PageBuffer(buffer, j * 8192, num3++)); } _extends++; } } public ICollection GetPages() { return _readable.Values; } public int Clear() { int num = 0; Constants.ENSURE(PagesInUse == 0, "must have no pages in use when call Clear() cache"); foreach (PageBuffer value in _readable.Values) { value.Position = long.MaxValue; value.Origin = FileOrigin.None; _free.Enqueue(value); num++; } _readable.Clear(); return num; } public void Dispose() { } } internal class BufferReader : IDisposable { private readonly IEnumerator _source; private readonly bool _utcDate; private BufferSlice _current; private int _currentPosition; private int _position; private bool _isEOF; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public int Position => _position; public bool IsEOF => _isEOF; public BufferReader(byte[] buffer, bool utcDate = false) : this(new BufferSlice(buffer, 0, buffer.Length), utcDate) { } public BufferReader(BufferSlice buffer, bool utcDate = false) { _source = null; _utcDate = utcDate; _current = buffer; } public BufferReader(IEnumerable source, bool utcDate = false) { _source = source.GetEnumerator(); _utcDate = utcDate; _source.MoveNext(); _current = _source.Current; } private bool MoveForward(int count) { if (_isEOF) { return false; } Constants.ENSURE(_currentPosition + count <= _current.Count, "forward is only for current segment"); _currentPosition += count; _position += count; if (_currentPosition == _current.Count) { if (_source == null || !_source.MoveNext()) { _isEOF = true; } else { _current = _source.Current; _currentPosition = 0; } return true; } return false; } public int Read(byte[] buffer, int offset, int count) { int num = 0; while (num < count) { int val = _current.Count - _currentPosition; int num2 = Math.Min(count - num, val); if (buffer != null) { Buffer.BlockCopy(_current.Array, _current.Offset + _currentPosition, buffer, offset + num, num2); } num += num2; MoveForward(num2); if (_isEOF) { break; } } Constants.ENSURE(count == num, "current value must fit inside defined buffer"); return num; } public int Skip(int count) { return Read(null, 0, count); } public void Consume() { if (_source != null) { while (_source.MoveNext()) { } } } public string ReadString(int count) { string @string; if (_currentPosition + count <= _current.Count) { @string = StringEncoding.UTF8.GetString(_current.Array, _current.Offset + _currentPosition, count); MoveForward(count); } else { byte[] array = _bufferPool.Rent(count); Read(array, 0, count); @string = StringEncoding.UTF8.GetString(array, 0, count); _bufferPool.Return(array, clearArray: true); } return @string; } public string ReadCString() { if (TryReadCStringCurrentSegment(out var value)) { return value; } using MemoryStream memoryStream = new MemoryStream(); int count = _current.Count - _currentPosition; memoryStream.Write(_current.Array, _current.Offset + _currentPosition, count); MoveForward(count); while (_current[_currentPosition] != 0 && !_isEOF) { memoryStream.WriteByte(_current[_currentPosition]); MoveForward(1); } MoveForward(1); return StringEncoding.UTF8.GetString(memoryStream.ToArray()); } private bool TryReadCStringCurrentSegment(out string value) { int i = _currentPosition; int num = 0; for (; i < _current.Count; i++) { if (_current[i] == 0) { value = StringEncoding.UTF8.GetString(_current.Array, _current.Offset + _currentPosition, num); MoveForward(num + 1); return true; } num++; } value = null; return false; } private T ReadNumber(Func convert, int size) { T result; if (_currentPosition + size <= _current.Count) { result = convert(_current.Array, _current.Offset + _currentPosition); MoveForward(size); } else { byte[] array = _bufferPool.Rent(size); Read(array, 0, size); result = convert(array, 0); _bufferPool.Return(array, clearArray: true); } return result; } public int ReadInt32() { return ReadNumber(BitConverter.ToInt32, 4); } public long ReadInt64() { return ReadNumber(BitConverter.ToInt64, 8); } public uint ReadUInt32() { return ReadNumber(BitConverter.ToUInt32, 4); } public double ReadDouble() { return ReadNumber(BitConverter.ToDouble, 8); } public decimal ReadDecimal() { int num = ReadInt32(); int num2 = ReadInt32(); int num3 = ReadInt32(); int num4 = ReadInt32(); return new decimal(new int[4] { num, num2, num3, num4 }); } public DateTime ReadDateTime() { DateTime result = new DateTime(ReadInt64(), DateTimeKind.Utc); if (!_utcDate) { return result; } return result.ToLocalTime(); } public Guid ReadGuid() { Guid result; if (_currentPosition + 16 <= _current.Count) { result = _current.ReadGuid(_currentPosition); MoveForward(16); } else { result = new Guid(ReadBytes(16)); } return result; } public ObjectId ReadObjectId() { ObjectId result; if (_currentPosition + 12 <= _current.Count) { result = new ObjectId(_current.Array, _current.Offset + _currentPosition); MoveForward(12); } else { byte[] array = _bufferPool.Rent(12); Read(array, 0, 12); result = new ObjectId(array); _bufferPool.Return(array, clearArray: true); } return result; } public bool ReadBoolean() { bool result = _current[_currentPosition] != 0; MoveForward(1); return result; } public byte ReadByte() { byte result = _current[_currentPosition]; MoveForward(1); return result; } internal PageAddress ReadPageAddress() { return new PageAddress(ReadUInt32(), ReadByte()); } public byte[] ReadBytes(int count) { byte[] array = new byte[count]; Read(array, 0, count); return array; } public BsonValue ReadIndexKey() { return (BsonType)ReadByte() switch { BsonType.Null => BsonValue.Null, BsonType.Int32 => ReadInt32(), BsonType.Int64 => ReadInt64(), BsonType.Double => ReadDouble(), BsonType.Decimal => ReadDecimal(), BsonType.String => ReadString(ReadByte()), BsonType.Document => ReadDocument().GetValue(), BsonType.Array => ReadArray().GetValue(), BsonType.Binary => ReadBytes(ReadByte()), BsonType.ObjectId => ReadObjectId(), BsonType.Guid => ReadGuid(), BsonType.Boolean => ReadBoolean(), BsonType.DateTime => ReadDateTime(), BsonType.MinValue => BsonValue.MinValue, BsonType.MaxValue => BsonValue.MaxValue, _ => throw new NotImplementedException(), }; } public Result ReadDocument(HashSet fields = null) { BsonDocument bsonDocument = new BsonDocument(); try { int num = ReadInt32(); int num2 = _position + num - 5; HashSet hashSet = ((fields == null || fields.Count == 0) ? null : new HashSet(fields, StringComparer.OrdinalIgnoreCase)); while (_position < num2 && (hashSet == null || (hashSet != null && hashSet.Count > 0))) { string name; BsonValue bsonValue = ReadElement(hashSet, out name); if (bsonValue != null) { bsonDocument[name] = bsonValue; hashSet?.Remove(name); } } MoveForward(1); return bsonDocument; } catch (Exception ex) { return new Result(bsonDocument, ex); } } public Result ReadArray() { BsonArray bsonArray = new BsonArray(); try { int num = ReadInt32(); int num2 = _position + num - 5; while (_position < num2) { string name; BsonValue item = ReadElement(null, out name); bsonArray.Add(item); } MoveForward(1); return bsonArray; } catch (Exception ex) { return new Result(bsonArray, ex); } } private BsonValue ReadElement(HashSet remaining, out string name) { byte b = ReadByte(); name = ReadCString(); if (remaining != null && !remaining.Contains(name)) { int num; switch (b) { default: num = 0; break; case 3: case 4: num = ReadInt32() - 4; break; case 5: num = ReadInt32() + 1; break; case 2: num = ReadInt32(); break; case 19: num = 16; break; case 7: num = 12; break; case 1: case 9: case 18: num = 8; break; case 16: num = 4; break; case 8: num = 1; break; case 10: case 127: case byte.MaxValue: num = 0; break; } int num2 = num; if (num2 > 0) { Skip(num2); } return null; } switch (b) { case 1: return ReadDouble(); case 2: { int num4 = ReadInt32(); string text = ReadString(num4 - 1); MoveForward(1); return text; } case 3: return ReadDocument().GetValue(); case 4: return ReadArray().GetValue(); case 5: { int count = ReadInt32(); byte b2 = ReadByte(); byte[] array = ReadBytes(count); switch (b2) { case 0: return array; case 4: return new Guid(array); } break; } case 7: return ReadObjectId(); case 8: return ReadBoolean(); case 9: { long num3 = ReadInt64(); switch (num3) { case 253402300800000L: return DateTime.MaxValue; case -62135596800000L: return DateTime.MinValue; default: { DateTime unixEpoch = BsonValue.UnixEpoch; DateTime dateTime = unixEpoch.AddMilliseconds(num3); return _utcDate ? dateTime : dateTime.ToLocalTime(); } } } case 10: return BsonValue.Null; case 16: return ReadInt32(); case 18: return ReadInt64(); case 19: return ReadDecimal(); case byte.MaxValue: return BsonValue.MinValue; case 127: return BsonValue.MaxValue; } throw new NotSupportedException("BSON type not supported"); } public void Dispose() { _source?.Dispose(); } } internal class BufferWriter : IDisposable { private readonly IEnumerator _source; private BufferSlice _current; private int _currentPosition; private int _position; private bool _isEOF; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public int Position => _position; public bool IsEOF => _isEOF; public BufferWriter(byte[] buffer) : this(new BufferSlice(buffer, 0, buffer.Length)) { } public BufferWriter(BufferSlice buffer) { _source = null; _current = buffer; } public BufferWriter(IEnumerable source) { _source = source.GetEnumerator(); _source.MoveNext(); _current = _source.Current; } private bool MoveForward(int count) { if (_isEOF) { return false; } Constants.ENSURE(_currentPosition + count <= _current.Count, "forward is only for current segment"); _currentPosition += count; _position += count; if (_currentPosition == _current.Count) { if (_source == null || !_source.MoveNext()) { _isEOF = true; } else { _current = _source.Current; _currentPosition = 0; } return true; } return false; } public int Write(byte[] buffer, int offset, int count) { int num = 0; while (num < count) { int val = _current.Count - _currentPosition; int num2 = Math.Min(count - num, val); if (buffer != null) { Buffer.BlockCopy(buffer, offset + num, _current.Array, _current.Offset + _currentPosition, num2); } num += num2; MoveForward(num2); if (_isEOF) { break; } } Constants.ENSURE(count == num, "current value must fit inside defined buffer"); return num; } public int Write(byte[] buffer) { return Write(buffer, 0, buffer.Length); } public int Skip(int count) { return Write(null, 0, count); } public void Consume() { if (_source != null) { while (_source.MoveNext()) { } } } public void WriteCString(string value) { if (value.IndexOf('\0') > -1) { throw LiteException.InvalidNullCharInString(); } int byteCount = StringEncoding.UTF8.GetByteCount(value); int num = _current.Count - _currentPosition; if (byteCount < num) { StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); _current[_currentPosition + byteCount] = 0; MoveForward(byteCount + 1); return; } byte[] array = _bufferPool.Rent(byteCount); StringEncoding.UTF8.GetBytes(value, 0, value.Length, array, 0); Write(array, 0, byteCount); _current[_currentPosition] = 0; MoveForward(1); _bufferPool.Return(array, clearArray: true); } public void WriteString(string value, bool specs) { int byteCount = StringEncoding.UTF8.GetByteCount(value); if (specs) { Write(byteCount + 1); } if (byteCount <= _current.Count - _currentPosition) { StringEncoding.UTF8.GetBytes(value, 0, value.Length, _current.Array, _current.Offset + _currentPosition); MoveForward(byteCount); } else { byte[] array = _bufferPool.Rent(byteCount); StringEncoding.UTF8.GetBytes(value, 0, value.Length, array, 0); Write(array, 0, byteCount); _bufferPool.Return(array, clearArray: true); } if (specs) { Write((byte)0); } } private void WriteNumber(T value, Action toBytes, int size) { if (_currentPosition + size <= _current.Count) { toBytes(value, _current.Array, _current.Offset + _currentPosition); MoveForward(size); return; } byte[] array = _bufferPool.Rent(size); toBytes(value, array, 0); Write(array, 0, size); _bufferPool.Return(array, clearArray: true); } public void Write(int value) { WriteNumber(value, BufferExtensions.ToBytes, 4); } public void Write(long value) { WriteNumber(value, BufferExtensions.ToBytes, 8); } public void Write(uint value) { WriteNumber(value, BufferExtensions.ToBytes, 4); } public void Write(double value) { WriteNumber(value, BufferExtensions.ToBytes, 8); } public void Write(decimal value) { int[] bits = decimal.GetBits(value); Write(bits[0]); Write(bits[1]); Write(bits[2]); Write(bits[3]); } public void Write(DateTime value) { Write(((value == DateTime.MinValue || value == DateTime.MaxValue) ? value : value.ToUniversalTime()).Ticks); } public void Write(Guid value) { byte[] buffer = value.ToByteArray(); Write(buffer, 0, 16); } public void Write(ObjectId value) { if (_currentPosition + 12 <= _current.Count) { value.ToByteArray(_current.Array, _current.Offset + _currentPosition); MoveForward(12); return; } byte[] array = _bufferPool.Rent(12); value.ToByteArray(array, 0); Write(array, 0, 12); _bufferPool.Return(array, clearArray: true); } public void Write(bool value) { _current[_currentPosition] = (value ? ((byte)1) : ((byte)0)); MoveForward(1); } public void Write(byte value) { _current[_currentPosition] = value; MoveForward(1); } internal void Write(PageAddress address) { Write(address.PageID); Write(address.Index); } public int WriteArray(BsonArray value, bool recalc) { int bytesCount = value.GetBytesCount(recalc); Write(bytesCount); for (int i = 0; i < value.Count; i++) { WriteElement(i.ToString(), value[i]); } Write((byte)0); return bytesCount; } public int WriteDocument(BsonDocument value, bool recalc) { int bytesCount = value.GetBytesCount(recalc); Write(bytesCount); foreach (KeyValuePair element in value.GetElements()) { WriteElement(element.Key, element.Value); } Write((byte)0); return bytesCount; } private void WriteElement(string key, BsonValue value) { switch (value.Type) { case BsonType.Double: Write((byte)1); WriteCString(key); Write(value.AsDouble); break; case BsonType.String: Write((byte)2); WriteCString(key); WriteString(value.AsString, specs: true); break; case BsonType.Document: Write((byte)3); WriteCString(key); WriteDocument(value.AsDocument, recalc: false); break; case BsonType.Array: Write((byte)4); WriteCString(key); WriteArray(value.AsArray, recalc: false); break; case BsonType.Binary: { Write((byte)5); WriteCString(key); byte[] asBinary = value.AsBinary; Write(asBinary.Length); Write((byte)0); Write(asBinary, 0, asBinary.Length); break; } case BsonType.Guid: { Write((byte)5); WriteCString(key); Guid asGuid = value.AsGuid; Write(16); Write((byte)4); Write(asGuid); break; } case BsonType.ObjectId: Write((byte)7); WriteCString(key); Write(value.AsObjectId); break; case BsonType.Boolean: Write((byte)8); WriteCString(key); Write(value.AsBoolean ? ((byte)1) : ((byte)0)); break; case BsonType.DateTime: { Write((byte)9); WriteCString(key); DateTime asDateTime = value.AsDateTime; Write(Convert.ToInt64((((asDateTime == DateTime.MinValue || asDateTime == DateTime.MaxValue) ? asDateTime : asDateTime.ToUniversalTime()) - BsonValue.UnixEpoch).TotalMilliseconds)); break; } case BsonType.Null: Write((byte)10); WriteCString(key); break; case BsonType.Int32: Write((byte)16); WriteCString(key); Write(value.AsInt32); break; case BsonType.Int64: Write((byte)18); WriteCString(key); Write(value.AsInt64); break; case BsonType.Decimal: Write((byte)19); WriteCString(key); Write(value.AsDecimal); break; case BsonType.MinValue: Write(byte.MaxValue); WriteCString(key); break; case BsonType.MaxValue: Write((byte)127); WriteCString(key); break; } } public void Dispose() { _source?.Dispose(); } } internal class FileStreamFactory : IStreamFactory { private readonly string _filename; private readonly string _password; private readonly bool _readonly; private readonly bool _hidden; private readonly bool _useAesStream; public string Name => Path.GetFileName(_filename); public bool CloseOnDispose => true; public FileStreamFactory(string filename, string password, bool readOnly, bool hidden, bool useAesStream = true) { _filename = filename; _password = password; _readonly = readOnly; _hidden = hidden; _useAesStream = useAesStream; } public Stream GetStream(bool canWrite, bool sequencial) { bool num = canWrite && !_readonly; FileMode mode = (_readonly ? FileMode.Open : FileMode.OpenOrCreate); FileAccess access = ((!num) ? FileAccess.Read : FileAccess.ReadWrite); FileShare share = (num ? FileShare.Read : FileShare.ReadWrite); FileOptions options = (sequencial ? FileOptions.SequentialScan : FileOptions.RandomAccess); bool num2 = num && !Exists(); FileStream fileStream = new FileStream(_filename, mode, access, share, 8192, options); if (num2 && _hidden) { File.SetAttributes(_filename, FileAttributes.Hidden); } if (_password != null && _useAesStream) { return new AesStream(_password, fileStream); } return fileStream; } public long GetLength() { if (!Exists()) { return 0L; } long num = new FileInfo(_filename).Length; if (num % 8192 != 0L) { num -= num % 8192; using FileStream fileStream = new FileStream(_filename, FileMode.Open, FileAccess.Write, FileShare.None, 8192, FileOptions.SequentialScan); fileStream.SetLength(num); fileStream.FlushToDisk(); } if (num <= 0) { return 0L; } return num - ((_password != null) ? 8192 : 0); } public bool Exists() { return File.Exists(_filename); } public void Delete() { File.Delete(_filename); } public bool IsLocked() { if (Exists()) { return FileHelper.IsFileLocked(_filename); } return false; } } internal interface IStreamFactory { string Name { get; } bool CloseOnDispose { get; } Stream GetStream(bool canWrite, bool sequencial); long GetLength(); bool Exists(); void Delete(); bool IsLocked(); } internal class StreamFactory : IStreamFactory { private readonly Stream _stream; private readonly string _password; public string Name { get { if (!(_stream is MemoryStream)) { if (!(_stream is TempStream)) { return ":stream:"; } return ":temp:"; } return ":memory:"; } } public bool CloseOnDispose => false; public StreamFactory(Stream stream, string password) { _stream = stream; _password = password; } public Stream GetStream(bool canWrite, bool sequencial) { if (_password == null) { return new ConcurrentStream(_stream, canWrite); } return new AesStream(_password, new ConcurrentStream(_stream, canWrite)); } public long GetLength() { long num = _stream.Length; if (num % 8192 != 0L) { num -= num % 8192; _stream.SetLength(num); _stream.FlushToDisk(); } if (num <= 0) { return 0L; } return num - ((_password != null) ? 8192 : 0); } public bool Exists() { return _stream.Length > 0; } public void Delete() { } public bool IsLocked() { return false; } } internal class StreamPool : IDisposable { private readonly ConcurrentBag _pool = new ConcurrentBag(); private readonly Lazy _writer; private readonly IStreamFactory _factory; public Lazy Writer => _writer; public StreamPool(IStreamFactory factory, bool appendOnly) { StreamPool streamPool = this; _factory = factory; _writer = new Lazy(() => streamPool._factory.GetStream(canWrite: true, appendOnly), isThreadSafe: true); } public Stream Rent() { if (!_pool.TryTake(out var result)) { return _factory.GetStream(canWrite: false, sequencial: false); } return result; } public void Return(Stream stream) { _pool.Add(stream); } public void Dispose() { if (!_factory.CloseOnDispose) { return; } foreach (Stream item in _pool) { item.Dispose(); } if (_writer.IsValueCreated) { _writer.Value.Dispose(); } } } public class AesStream : Stream { private readonly Aes _aes; private readonly ICryptoTransform _encryptor; private readonly ICryptoTransform _decryptor; private readonly string _name; private readonly Stream _stream; private readonly CryptoStream _reader; private readonly CryptoStream _writer; private readonly byte[] _decryptedZeroes = new byte[16]; private static readonly byte[] _emptyContent = new byte[8175]; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public byte[] Salt { get; } public override bool CanRead => _stream.CanRead; public override bool CanSeek => _stream.CanSeek; public override bool CanWrite => _stream.CanWrite; public override long Length => _stream.Length - 8192; public override long Position { get { return _stream.Position - 8192; } set { Seek(value, SeekOrigin.Begin); } } public long StreamPosition => _stream.Position; public AesStream(string password, Stream stream) { _stream = stream; _name = ((_stream is FileStream fileStream) ? Path.GetFileName(fileStream.Name) : null); bool flag = _stream.Length < 8192; _stream.Position = 0L; byte[] array = _bufferPool.Rent(32); byte[] array2 = _bufferPool.Rent(16); try { if (flag) { Salt = NewSalt(); _stream.WriteByte(1); _stream.Write(Salt, 0, 16); } else { Salt = new byte[16]; if (_stream.ReadByte() != 1) { throw LiteException.FileNotEncrypted(); } _stream.Read(Salt, 0, 16); } _aes = Aes.Create(); _aes.Padding = PaddingMode.None; _aes.Mode = CipherMode.ECB; Rfc2898DeriveBytes rfc2898DeriveBytes = new Rfc2898DeriveBytes(password, Salt); using (rfc2898DeriveBytes) { _aes.Key = rfc2898DeriveBytes.GetBytes(32); _aes.IV = rfc2898DeriveBytes.GetBytes(16); } _encryptor = _aes.CreateEncryptor(); _decryptor = _aes.CreateDecryptor(); _reader = (_stream.CanRead ? new CryptoStream(_stream, _decryptor, CryptoStreamMode.Read) : null); _writer = (_stream.CanWrite ? new CryptoStream(_stream, _encryptor, CryptoStreamMode.Write) : null); _stream.Position = 32L; if (!flag) { _stream.Read(array, 0, 32); flag = array.All((byte x) => x == 0); Array.Clear(array, 0, 32); _stream.Position = 32L; } if (flag) { array.Fill(1, 0, 32); _writer.Write(array, 0, 32); _stream.Position = 8191L; _stream.WriteByte(0); } else { _reader.Read(array, 0, 32); if (!array.All((byte x) => x == 1)) { throw LiteException.InvalidPassword(); } } _stream.Position = 8192L; _stream.FlushToDisk(); using MemoryStream stream2 = new MemoryStream(array2); using CryptoStream cryptoStream = new CryptoStream(stream2, _decryptor, CryptoStreamMode.Read); cryptoStream.Read(_decryptedZeroes, 0, _decryptedZeroes.Length); } catch { _stream.Dispose(); throw; } finally { _bufferPool.Return(array2, clearArray: true); _bufferPool.Return(array, clearArray: true); } } public override int Read(byte[] array, int offset, int count) { Constants.ENSURE(Position % 8192 == 0, "AesRead: position must be in PAGE_SIZE module. Position={0}, File={1}", Position, _name); int result = _reader.Read(array, offset, count); if (IsBlank(array, offset)) { array.Fill(0, offset, count); } return result; } public override void Write(byte[] array, int offset, int count) { Constants.ENSURE(count == 8192 || count == 1, "buffer size must be PAGE_SIZE"); Constants.ENSURE(Position == 191 || Position % 8192 == 0, "AesWrite: position must be in PAGE_SIZE module. Position={0}, File={1}", Position, _name); _writer.Write(array, offset, count); } protected override void Dispose(bool disposing) { base.Dispose(disposing); _stream?.Dispose(); _encryptor.Dispose(); _decryptor.Dispose(); _aes.Dispose(); } public static byte[] NewSalt() { byte[] array = new byte[16]; using RandomNumberGenerator randomNumberGenerator = RandomNumberGenerator.Create(); randomNumberGenerator.GetBytes(array); return array; } public override void Flush() { _stream.Flush(); } public override long Seek(long offset, SeekOrigin origin) { return _stream.Seek(offset + 8192, origin); } public override void SetLength(long value) { _stream.SetLength(value + 8192); } private unsafe bool IsBlank(byte[] array, int offset) { fixed (byte* ptr3 = array) { fixed (byte* ptr = _decryptedZeroes) { void* ptr2 = ptr; ulong* ptr4 = (ulong*)(ptr3 + offset); ulong* ptr5 = (ulong*)ptr2; if (*ptr4 == *ptr5) { return ptr4[1] == ptr5[1]; } return false; } } } } internal class ConcurrentStream : Stream { private readonly Stream _stream; private readonly bool _canWrite; private long _position; public override bool CanRead => _stream.CanRead; public override bool CanSeek => _stream.CanSeek; public override bool CanWrite => _canWrite; public override long Length => _stream.Length; public override long Position { get { return _position; } set { _position = value; } } public ConcurrentStream(Stream stream, bool canWrite) { _stream = stream; _canWrite = canWrite; } public override void Flush() { _stream.Flush(); } public override void SetLength(long value) { _stream.SetLength(value); } protected override void Dispose(bool disposing) { _stream.Dispose(); } public override long Seek(long offset, SeekOrigin origin) { lock (_stream) { _position = origin switch { SeekOrigin.Current => _position + offset, SeekOrigin.Begin => offset, _ => _position - offset, }; return _position; } } public override int Read(byte[] buffer, int offset, int count) { lock (_stream) { _stream.Position = _position; int result = _stream.Read(buffer, offset, count); _position = _stream.Position; return result; } } public override void Write(byte[] buffer, int offset, int count) { if (!_canWrite) { throw new NotSupportedException("Current stream are readonly"); } lock (_stream) { _stream.Position = _position; _stream.Write(buffer, offset, count); _position = _stream.Position; } } } public class TempStream : Stream { private Stream _stream = new MemoryStream(); private string _filename; private readonly long _maxMemoryUsage; public bool InMemory => _stream is MemoryStream; public bool InDisk => _stream is FileStream; public string Filename => _filename; public override bool CanRead => _stream.CanRead; public override bool CanSeek => _stream.CanWrite; public override bool CanWrite => _stream.CanWrite; public override long Length => _stream.Length; public override long Position { get { return _stream.Position; } set { Seek(value, SeekOrigin.Begin); } } public TempStream(string filename = null, long maxMemoryUsage = 10485760L) { _maxMemoryUsage = maxMemoryUsage; _filename = filename; } public override void Flush() { _stream.Flush(); } public override int Read(byte[] buffer, int offset, int count) { return _stream.Read(buffer, offset, count); } public override void SetLength(long value) { _stream.SetLength(value); } public override void Write(byte[] buffer, int offset, int count) { _stream.Write(buffer, offset, count); } public override long Seek(long offset, SeekOrigin origin) { if (origin switch { SeekOrigin.Current => _stream.Position + offset, SeekOrigin.Begin => offset, _ => _stream.Position - offset, } > _maxMemoryUsage && InMemory) { _filename = _filename ?? Path.Combine(Path.GetTempPath(), "litedb_" + Guid.NewGuid().ToString() + ".db"); FileStream fileStream = new FileStream(_filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 8192, FileOptions.RandomAccess); _stream.Position = 0L; _stream.CopyTo(fileStream); _stream.Dispose(); _stream = fileStream; } return _stream.Seek(offset, origin); } protected override void Dispose(bool disposing) { base.Dispose(disposing); _stream.Dispose(); if (InDisk) { File.Delete(_filename); } } } internal class EnginePragmas { public const int P_USER_VERSION = 76; public const int P_COLLATION_LCID = 80; public const int P_COLLATION_SORT = 84; public const int P_TIMEOUT = 88; public const int P_UTC_DATE = 96; public const int P_CHECKPOINT = 97; public const int P_LIMIT_SIZE = 101; private readonly Dictionary _pragmas; private bool _isDirty; private readonly HeaderPage _headerPage; public int UserVersion { get; private set; } public Collation Collation { get; private set; } = Collation.Default; public TimeSpan Timeout { get; private set; } = TimeSpan.FromMinutes(1.0); public long LimitSize { get; private set; } = long.MaxValue; public bool UtcDate { get; private set; } public int Checkpoint { get; private set; } = 1000; public IEnumerable Pragmas => _pragmas.Values; public EnginePragmas(HeaderPage headerPage) { _headerPage = headerPage; _pragmas = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["USER_VERSION"] = new Pragma { Name = "USER_VERSION", Get = () => UserVersion, Set = delegate(BsonValue v) { UserVersion = v.AsInt32; }, Read = delegate(BufferSlice b) { UserVersion = b.ReadInt32(76); }, Validate = delegate { }, Write = delegate(BufferSlice b) { b.Write(UserVersion, 76); } }, ["COLLATION"] = new Pragma { Name = "COLLATION", Get = () => Collation.ToString(), Set = delegate(BsonValue v) { Collation = new Collation(v.AsString); }, Read = delegate(BufferSlice b) { Collation = new Collation(b.ReadInt32(80), (CompareOptions)b.ReadInt32(84)); }, Validate = delegate { throw new LiteException(0, "Pragma COLLATION is read only. Use Rebuild options."); }, Write = delegate(BufferSlice b) { b.Write(Collation.LCID, 80); b.Write((int)Collation.SortOptions, 84); } }, ["TIMEOUT"] = new Pragma { Name = "TIMEOUT", Get = () => (int)Timeout.TotalSeconds, Set = delegate(BsonValue v) { Timeout = TimeSpan.FromSeconds(v.AsInt32); }, Read = delegate(BufferSlice b) { Timeout = TimeSpan.FromSeconds(b.ReadInt32(88)); }, Validate = delegate(BsonValue v, HeaderPage h) { if (v <= 0) { throw new LiteException(0, "Pragma TIMEOUT must be greater than zero"); } }, Write = delegate(BufferSlice b) { b.Write((int)Timeout.TotalSeconds, 88); } }, ["LIMIT_SIZE"] = new Pragma { Name = "LIMIT_SIZE", Get = () => LimitSize, Set = delegate(BsonValue v) { LimitSize = v.AsInt64; }, Read = delegate(BufferSlice b) { long num = b.ReadInt64(101); LimitSize = ((num == 0L) ? long.MaxValue : num); }, Validate = delegate(BsonValue v, HeaderPage h) { if (v < 32768) { throw new LiteException(0, "Pragma LIMIT_SIZE must be at least 4 pages (32768 bytes)"); } if (h != null && v.AsInt64 < (h.LastPageID + 1) * 8192) { throw new LiteException(0, "Pragma LIMIT_SIZE must be greater or equal to the current file size"); } }, Write = delegate(BufferSlice b) { b.Write(LimitSize, 101); } }, ["UTC_DATE"] = new Pragma { Name = "UTC_DATE", Get = () => UtcDate, Set = delegate(BsonValue v) { UtcDate = v.AsBoolean; }, Read = delegate(BufferSlice b) { UtcDate = b.ReadBool(96); }, Validate = delegate { }, Write = delegate(BufferSlice b) { b.Write(UtcDate, 96); } }, ["CHECKPOINT"] = new Pragma { Name = "CHECKPOINT", Get = () => Checkpoint, Set = delegate(BsonValue v) { Checkpoint = v.AsInt32; }, Read = delegate(BufferSlice b) { Checkpoint = b.ReadInt32(97); }, Validate = delegate(BsonValue v, HeaderPage h) { if (v < 0) { throw new LiteException(0, "Pragma CHECKPOINT must be greater or equal to zero"); } }, Write = delegate(BufferSlice b) { b.Write(Checkpoint, 97); } } }; _isDirty = true; } public EnginePragmas(BufferSlice buffer, HeaderPage headerPage) : this(headerPage) { foreach (Pragma value in _pragmas.Values) { value.Read(buffer); } _isDirty = false; } public void UpdateBuffer(BufferSlice buffer) { if (!_isDirty) { return; } foreach (KeyValuePair pragma in _pragmas) { pragma.Value.Write(buffer); } _isDirty = false; } public BsonValue Get(string name) { if (_pragmas.TryGetValue(name, out var value)) { return value.Get(); } throw new LiteException(0, "Pragma `" + name + "` not exist"); } public void Set(string name, BsonValue value, bool validate) { if (_pragmas.TryGetValue(name, out var value2)) { if (validate) { value2.Validate(value, _headerPage); } value2.Set(value); _isDirty = true; return; } throw new LiteException(0, "Pragma `" + name + "` not exist"); } } public class EngineSettings { public Stream DataStream { get; set; } public Stream LogStream { get; set; } public Stream TempStream { get; set; } public string Filename { get; set; } public string Password { get; set; } public long InitialSize { get; set; } public Collation Collation { get; set; } public bool ReadOnly { get; set; } public bool AutoRebuild { get; set; } public bool Upgrade { get; set; } public Func ReadTransform { get; set; } internal IStreamFactory CreateDataFactory(bool useAesStream = true) { if (DataStream != null) { return new StreamFactory(DataStream, Password); } if (Filename == ":memory:") { return new StreamFactory(new MemoryStream(), Password); } if (Filename == ":temp:") { return new StreamFactory(new TempStream(null, 10485760L), Password); } if (!string.IsNullOrEmpty(Filename)) { return new FileStreamFactory(Filename, Password, ReadOnly, hidden: false, useAesStream); } throw new ArgumentException("EngineSettings must have Filename or DataStream as data source"); } internal IStreamFactory CreateLogFactory() { if (LogStream != null) { return new StreamFactory(LogStream, Password); } if (Filename == ":memory:") { return new StreamFactory(new MemoryStream(), Password); } if (Filename == ":temp:") { return new StreamFactory(new TempStream(null, 10485760L), Password); } if (!string.IsNullOrEmpty(Filename)) { return new FileStreamFactory(FileHelper.GetLogFile(Filename), Password, ReadOnly, hidden: false); } return new StreamFactory(new MemoryStream(), Password); } internal IStreamFactory CreateTempFactory() { if (TempStream != null) { return new StreamFactory(TempStream, Password); } if (Filename == ":memory:") { return new StreamFactory(new MemoryStream(), Password); } if (Filename == ":temp:") { return new StreamFactory(new TempStream(null, 10485760L), Password); } if (!string.IsNullOrEmpty(Filename)) { return new FileStreamFactory(FileHelper.GetTempFile(Filename), Password, readOnly: false, hidden: true); } return new StreamFactory(new TempStream(null, 10485760L), Password); } } internal class EngineState { public bool Disposed; private Exception _exception; private readonly LiteEngine _engine; private readonly EngineSettings _settings; public EngineState(LiteEngine engine, EngineSettings settings) { _engine = engine; _settings = settings; } public void Validate() { if (Disposed) { throw _exception ?? LiteException.EngineDisposed(); } } public bool Handle(Exception ex) { if (ex is IOException || (ex is LiteException ex2 && ex2.ErrorCode == 999)) { _exception = ex; _engine?.Close(ex); return false; } return true; } public BsonValue ReadTransform(string collection, BsonValue value) { if (_settings?.ReadTransform == null) { return value; } return _settings.ReadTransform(collection, value); } } public class LiteEngine : ILiteEngine, IDisposable { private static readonly ArrayPool _bufferPool = ArrayPool.Shared; private LockService _locker; private DiskService _disk; private WalIndexService _walIndex; private HeaderPage _header; private TransactionMonitor _monitor; private SortDisk _sortDisk; private EngineState _state; private readonly EngineSettings _settings; private Dictionary _systemCollections; private ConcurrentDictionary _sequences; public IEnumerable GetCollectionNames() { return from x in _header.GetCollections() select x.Key; } public bool DropCollection(string name) { if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } if (_locker.IsInTransaction) { throw LiteException.AlreadyExistsTransaction(); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, name, addIfNotExists: false); if (snapshot.CollectionPage == null) { return false; } snapshot.DropCollection(transaction.Safepoint); _sequences.TryRemove(name, out var _); return true; }); } public bool RenameCollection(string collection, string newName) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (newName.IsNullOrWhiteSpace()) { throw new ArgumentNullException("newName"); } if (_locker.IsInTransaction) { throw LiteException.AlreadyExistsTransaction(); } if (collection.Equals(newName, StringComparison.OrdinalIgnoreCase)) { throw LiteException.InvalidCollectionName(newName, "New name must be different from current name"); } CollectionService.CheckName(newName, _header); if (_locker.IsInTransaction) { throw LiteException.AlreadyExistsTransaction(); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: false); transaction.CreateSnapshot(LockMode.Write, newName, addIfNotExists: false); if (snapshot.CollectionPage == null) { return false; } if (_header.GetCollectionPageID(newName) != uint.MaxValue) { throw LiteException.AlreadyExistsCollectionName(newName); } transaction.Pages.Commit += delegate(HeaderPage h) { h.RenameCollection(collection, newName); }; return true; }); } public int Delete(string collection, IEnumerable ids) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (ids == null) { throw new ArgumentNullException("ids"); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: false); CollectionPage collectionPage = snapshot.CollectionPage; DataService dataService = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); IndexService indexService = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); if (collectionPage == null) { return 0; } int num = 0; CollectionIndex pK = collectionPage.PK; foreach (BsonValue id in ids) { IndexNode indexNode = indexService.Find(pK, id, sibling: false, 1); if (indexNode != null) { _state.Validate(); dataService.Delete(indexNode.DataBlock); indexService.DeleteAll(indexNode.Position); transaction.Safepoint(); num++; } } return num; }); } public int DeleteMany(string collection, BsonExpression predicate) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (predicate != null && predicate.Type == BsonExpressionType.Equal && predicate.Left.Type == BsonExpressionType.Path && predicate.Left.Source == "$._id" && predicate.Right.IsValue) { BsonValue bsonValue = predicate.Right.Execute(_header.Pragmas.Collation).First(); return Delete(collection, new BsonValue[1] { bsonValue }); } return Delete(collection, getIds()); IEnumerable getIds() { Query query = new Query { Select = "{ i: _id }", ForUpdate = true }; if (predicate != null) { query.Where.Add(predicate); } using IBsonDataReader reader = Query(collection, query); while (reader.Read()) { BsonValue bsonValue2 = reader.Current["i"]; if (bsonValue2 != BsonValue.Null) { yield return bsonValue2; } } } } public bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } if (expression == null) { throw new ArgumentNullException("expression"); } if (!expression.IsIndexable) { throw new ArgumentException("Index expressions must contains at least one document field. Used methods must be immutable. Parameters are not supported.", "expression"); } if (name.Length > Constants.INDEX_NAME_MAX_LENGTH) { throw LiteException.InvalidIndexName(name, collection, "MaxLength = " + Constants.INDEX_NAME_MAX_LENGTH); } if (!name.IsWord()) { throw LiteException.InvalidIndexName(name, collection, "Use only [a-Z$_]"); } if (name.StartsWith("$")) { throw LiteException.InvalidIndexName(name, collection, "Index name can't start with `$`"); } if (!expression.IsScalar && unique) { throw new LiteException(0, "Multikey index expression do not support unique option"); } if (expression.Source == "$._id") { return false; } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: true); CollectionPage collectionPage = snapshot.CollectionPage; IndexService indexService = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); DataService dataService = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); CollectionIndex collectionIndex = collectionPage.GetCollectionIndex(name); if (collectionIndex != null) { if (collectionIndex.Expression != expression.Source) { throw LiteException.IndexAlreadyExist(name); } return false; } CollectionIndex index = indexService.CreateIndex(name, expression.Source, unique); uint num = 0u; foreach (IndexNode item in new IndexAll("_id", 1).Run(collectionPage, indexService)) { using (BufferReader bufferReader = new BufferReader(dataService.Read(item.DataBlock))) { BsonDocument value = bufferReader.ReadDocument(expression.Fields).GetValue(); IndexNode indexNode = null; IndexNode indexNode2 = null; foreach (BsonValue indexKey in expression.GetIndexKeys(value, _header.Pragmas.Collation)) { _state.Validate(); IndexNode indexNode3 = indexService.AddNode(index, indexKey, item.DataBlock, indexNode); if (indexNode2 == null) { indexNode2 = indexNode3; } indexNode = indexNode3; num++; } if (indexNode2 != null) { indexNode.SetNextNode(item.NextNode); item.SetNextNode(indexNode2.Position); } } transaction.Safepoint(); } return true; }); } public bool DropIndex(string collection, string name) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (name.IsNullOrWhiteSpace()) { throw new ArgumentNullException("name"); } if (name == "_id") { throw LiteException.IndexDropId(); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: false); CollectionPage collectionPage = snapshot.CollectionPage; IndexService indexService = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); if (collectionPage == null) { return false; } CollectionIndex collectionIndex = collectionPage.GetCollectionIndex(name); if (collectionIndex == null) { return false; } indexService.DropIndex(collectionIndex); snapshot.CollectionPage.DeleteCollectionIndex(name); return true; }); } public int Insert(string collection, IEnumerable docs, BsonAutoId autoId) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (docs == null) { throw new ArgumentNullException("docs"); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: true); int num = 0; IndexService indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); DataService data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); foreach (BsonDocument doc in docs) { _state.Validate(); transaction.Safepoint(); InsertDocument(snapshot, doc, autoId, indexer, data); num++; } return num; }); } private void InsertDocument(Snapshot snapshot, BsonDocument doc, BsonAutoId autoId, IndexService indexer, DataService data) { if (!doc.TryGetValue("_id", out var value)) { value = (doc["_id"] = autoId switch { BsonAutoId.Guid => new BsonValue(Guid.NewGuid()), BsonAutoId.ObjectId => new BsonValue(ObjectId.NewObjectId()), _ => GetSequence(snapshot, autoId), }); } else if (value.IsNumber) { SetSequence(snapshot, value); } if (value.IsNull || value.IsMinValue || value.IsMaxValue) { throw LiteException.InvalidDataType("_id", value); } PageAddress dataBlock = data.Insert(doc); IndexNode last = null; foreach (CollectionIndex collectionIndex in snapshot.CollectionPage.GetCollectionIndexes()) { foreach (BsonValue indexKey in collectionIndex.BsonExpr.GetIndexKeys(doc, _header.Pragmas.Collation)) { last = indexer.AddNode(collectionIndex, indexKey, dataBlock, last); } } } public BsonValue Pragma(string name) { return _header.Pragmas.Get(name); } public bool Pragma(string name, BsonValue value) { if (Pragma(name) == value) { return false; } if (_locker.IsInTransaction) { throw LiteException.AlreadyExistsTransaction(); } return AutoTransaction(delegate(TransactionService transaction) { transaction.Pages.Commit += delegate(HeaderPage h) { h.Pragmas.Set(name, value, validate: true); }; return true; }); } public IBsonDataReader Query(string collection, Query query) { if (string.IsNullOrWhiteSpace(collection)) { throw new ArgumentNullException("collection"); } if (query == null) { throw new ArgumentNullException("query"); } IEnumerable source = null; if (collection.StartsWith("$")) { SqlParser.ParseCollection(new Tokenizer(collection), out var name, out var options); SystemCollection systemCollection = GetSystemCollection(name); source = systemCollection.Input(options); collection = systemCollection.Name; } return new QueryExecutor(this, _state, _monitor, _sortDisk, _disk, _header.Pragmas, collection, query, source).ExecuteQuery(); } public long Rebuild(RebuildOptions options) { if (string.IsNullOrEmpty(_settings.Filename)) { return 0L; } Close(); long result = new RebuildService(_settings).Rebuild(options); Open(); _state.Disposed = false; return result; } public long Rebuild() { Collation collation = new Collation(Pragma("COLLATION")); string password = _settings.Password; return Rebuild(new RebuildOptions { Password = password, Collation = collation }); } internal void RebuildContent(IFileReader reader) { bool isNew; TransactionService transaction = _monitor.GetTransaction(create: true, queryOnly: false, out isNew); try { foreach (string collection in reader.GetCollections()) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: true); IndexService indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); DataService data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); foreach (BsonDocument document in reader.GetDocuments(collection)) { transaction.Safepoint(); InsertDocument(snapshot, document, BsonAutoId.ObjectId, indexer, data); } foreach (IndexInfo index in reader.GetIndexes(collection)) { EnsureIndex(collection, index.Name, BsonExpression.Create(index.Expression), index.Unique); } } transaction.Commit(); _monitor.ReleaseTransaction(transaction); } catch (Exception ex) { Close(ex); throw; } } private void Recovery(Collation collation) { RebuildService rebuildService = new RebuildService(_settings); RebuildOptions options = new RebuildOptions { Collation = collation, Password = _settings.Password, IncludeErrorReport = true }; rebuildService.Rebuild(options); } private BsonValue GetSequence(Snapshot snapshot, BsonAutoId autoId) { long num = _sequences.AddOrUpdate(snapshot.CollectionName, delegate { BsonValue lastId = GetLastId(snapshot); if (lastId.IsMinValue) { return 1L; } if (!lastId.IsNumber) { throw new LiteException(0, $"It's not possible use AutoId={autoId} because '{snapshot.CollectionName}' collection contains not only numbers in _id index ({lastId})."); } return lastId.AsInt64 + 1; }, (string s, long value) => value + 1); if (autoId != BsonAutoId.Int32) { return new BsonValue(num); } return new BsonValue((int)num); } private void SetSequence(Snapshot snapshot, BsonValue newId) { _sequences.AddOrUpdate(snapshot.CollectionName, delegate { BsonValue lastId = GetLastId(snapshot); return lastId.IsNumber ? Math.Max(lastId.AsInt64, newId.AsInt64) : newId.AsInt64; }, (string s, long value) => Math.Max(value, newId.AsInt64)); } private BsonValue GetLastId(Snapshot snapshot) { CollectionIndex pK = snapshot.CollectionPage.PK; IndexPage page = snapshot.GetPage(pK.Tail.PageID); PageAddress pageAddress = page.GetIndexNode(pK.Tail.Index).Prev[0]; if (pageAddress == pK.Head) { return BsonValue.MinValue; } return ((pageAddress.PageID == page.PageID) ? page : snapshot.GetPage(pageAddress.PageID)).GetIndexNode(pageAddress.Index).Key; } internal SystemCollection GetSystemCollection(string name) { if (_systemCollections.TryGetValue(name, out var value)) { return value; } throw new LiteException(0, "System collection '" + name + "' are not registered as system collection"); } internal void RegisterSystemCollection(SystemCollection systemCollection) { if (systemCollection == null) { throw new ArgumentNullException("systemCollection"); } _systemCollections[systemCollection.Name] = systemCollection; } internal void RegisterSystemCollection(string collectionName, Func> factory) { if (collectionName.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collectionName"); } if (factory == null) { throw new ArgumentNullException("factory"); } _systemCollections[collectionName] = new SystemCollection(collectionName, factory); } public bool BeginTrans() { _state.Validate(); bool isNew; TransactionService transaction = _monitor.GetTransaction(create: true, queryOnly: false, out isNew); transaction.ExplicitTransaction = true; if (transaction.OpenCursors.Count > 0) { throw new LiteException(0, "This thread contains an open cursors/query. Close cursors before Begin()"); } return isNew; } public bool Commit() { _state.Validate(); bool isNew; TransactionService transaction = _monitor.GetTransaction(create: false, queryOnly: false, out isNew); if (transaction != null) { if (transaction.OpenCursors.Count > 0) { throw new LiteException(0, "Current transaction contains open cursors. Close cursors before run Commit()"); } if (transaction.State == TransactionState.Active) { CommitAndReleaseTransaction(transaction); return true; } } return false; } public bool Rollback() { _state.Validate(); bool isNew; TransactionService transaction = _monitor.GetTransaction(create: false, queryOnly: false, out isNew); if (transaction != null && transaction.State == TransactionState.Active) { transaction.Rollback(); _monitor.ReleaseTransaction(transaction); return true; } return false; } private T AutoTransaction(Func fn) { _state.Validate(); bool isNew; TransactionService transaction = _monitor.GetTransaction(create: true, queryOnly: false, out isNew); try { T result = fn(transaction); if (isNew) { CommitAndReleaseTransaction(transaction); } return result; } catch (Exception ex) { if (_state.Handle(ex)) { transaction.Rollback(); _monitor.ReleaseTransaction(transaction); } throw; } } private void CommitAndReleaseTransaction(TransactionService transaction) { transaction.Commit(); _monitor.ReleaseTransaction(transaction); if (_header.Pragmas.Checkpoint > 0 && _disk.GetFileLength(FileOrigin.Log) > _header.Pragmas.Checkpoint * 8192) { _walIndex.TryCheckpoint(); } } public int Update(string collection, IEnumerable docs) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (docs == null) { throw new ArgumentNullException("docs"); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: false); CollectionPage collectionPage = snapshot.CollectionPage; IndexService indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); DataService data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); int num = 0; if (collectionPage == null) { return 0; } foreach (BsonDocument doc in docs) { _state.Validate(); transaction.Safepoint(); if (UpdateDocument(snapshot, collectionPage, doc, indexer, data)) { num++; } } return num; }); } public int UpdateMany(string collection, BsonExpression transform, BsonExpression predicate) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (transform == null) { throw new ArgumentNullException("transform"); } return Update(collection, transformDocs()); IEnumerable transformDocs() { Query query = new Query { Select = "$", ForUpdate = true }; if (predicate != null) { query.Where.Add(predicate); } using IBsonDataReader reader = Query(collection, query); while (reader.Read()) { BsonDocument asDocument = reader.Current.AsDocument; BsonValue bsonValue = asDocument["_id"]; BsonValue bsonValue2 = transform.ExecuteScalar(asDocument, _header.Pragmas.Collation); if (!bsonValue2.IsDocument) { throw new ArgumentException("Extend expression must return a document", "transform"); } BsonDocument asDocument2 = BsonExpressionMethods.EXTEND(asDocument, bsonValue2.AsDocument).AsDocument; if (asDocument2.TryGetValue("_id", out var value)) { if (value != bsonValue) { throw LiteException.InvalidUpdateField("_id"); } } else { asDocument2["_id"] = bsonValue; } yield return asDocument2; } } } private bool UpdateDocument(Snapshot snapshot, CollectionPage col, BsonDocument doc, IndexService indexer, DataService data) { BsonValue bsonValue = doc["_id"]; if (bsonValue.IsNull || bsonValue.IsMinValue || bsonValue.IsMaxValue) { throw LiteException.InvalidDataType("_id", bsonValue); } IndexNode indexNode = indexer.Find(col.PK, bsonValue, sibling: false, 1); if (indexNode == null) { return false; } data.Update(col, indexNode.DataBlock, doc); Tuple[] oldKeys = (from x in indexer.GetNodeList(indexNode.NextNode) select new Tuple(x.Slot, x.Key, x.Position)).ToArray(); List> newKeys = new List>(); foreach (CollectionIndex item in from x in col.GetCollectionIndexes() where x.Name != "_id" select x) { foreach (BsonValue indexKey in item.BsonExpr.GetIndexKeys(doc, _header.Pragmas.Collation)) { newKeys.Add(new Tuple(item.Slot, indexKey, item.Name)); } } if (oldKeys.Length == 0 && newKeys.Count == 0) { return true; } HashSet hashSet = new HashSet(from x in oldKeys where !newKeys.Any((Tuple n) => n.Item1 == x.Item1 && n.Item2 == x.Item2) select x.Item3); Tuple[] array = newKeys.Where((Tuple x) => !oldKeys.Any((Tuple o) => o.Item1 == x.Item1 && o.Item2 == x.Item2)).ToArray(); if (hashSet.Count == 0 && array.Length == 0) { return true; } IndexNode last = indexer.DeleteList(indexNode.Position, hashSet); Tuple[] array2 = array; foreach (Tuple tuple in array2) { CollectionIndex collectionIndex = col.GetCollectionIndex(tuple.Item3); last = indexer.AddNode(collectionIndex, tuple.Item2, indexNode.DataBlock, last); } return true; } private void TryUpgrade() { if (!File.Exists(_settings.Filename)) { return; } byte[] array = _bufferPool.Rent(1024); using (FileStream fileStream = new FileStream(_settings.Filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024)) { fileStream.Position = 0L; fileStream.Read(array, 0, 1024); if (!FileReaderV7.IsVersion(array)) { return; } } _bufferPool.Return(array, clearArray: true); Recovery(_settings.Collation); } [Obsolete("Upgrade your LiteDB v4 datafiles using Upgrade=true in EngineSettings. You can use upgrade=true in connection string.")] public static bool Upgrade(string filename, string password = null, Collation collation = null) { if (filename.IsNullOrWhiteSpace()) { throw new ArgumentNullException("filename"); } if (!File.Exists(filename)) { return false; } using (new LiteEngine(new EngineSettings { Filename = filename, Password = password, Collation = collation, Upgrade = true })) { } return true; } public int Upsert(string collection, IEnumerable docs, BsonAutoId autoId) { if (collection.IsNullOrWhiteSpace()) { throw new ArgumentNullException("collection"); } if (docs == null) { throw new ArgumentNullException("docs"); } return AutoTransaction(delegate(TransactionService transaction) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Write, collection, addIfNotExists: true); CollectionPage collectionPage = snapshot.CollectionPage; IndexService indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); DataService data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT); int num = 0; foreach (BsonDocument doc in docs) { _state.Validate(); transaction.Safepoint(); if (doc["_id"] == BsonValue.Null || !UpdateDocument(snapshot, collectionPage, doc, indexer, data)) { InsertDocument(snapshot, doc, autoId, indexer, data); num++; } } return num; }); } public LiteEngine() : this(new EngineSettings { DataStream = new MemoryStream() }) { } public LiteEngine(string filename) : this(new EngineSettings { Filename = filename }) { } public LiteEngine(EngineSettings settings) { _settings = settings ?? throw new ArgumentNullException("settings"); Open(); } internal bool Open() { _systemCollections = new Dictionary(StringComparer.OrdinalIgnoreCase); _sequences = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); try { _state = new EngineState(this, _settings); if (_settings.Upgrade) { TryUpgrade(); } _disk = new DiskService(_settings, _state, Constants.MEMORY_SEGMENT_SIZES); PageBuffer pageBuffer = _disk.ReadFull(FileOrigin.Data).First(); if (pageBuffer[0] == 1) { throw new LiteException(0, "This data file is encrypted and needs a password to open"); } _header = new HeaderPage(pageBuffer); if (pageBuffer[191] != 0 && _settings.AutoRebuild) { _disk.Dispose(); _disk = null; Recovery(_header.Pragmas.Collation); _disk = new DiskService(_settings, _state, Constants.MEMORY_SEGMENT_SIZES); pageBuffer = _disk.ReadFull(FileOrigin.Data).First(); _header = new HeaderPage(pageBuffer); } if (_settings.Collation != null && _settings.Collation.ToString() != _header.Pragmas.Collation.ToString()) { throw new LiteException(0, $"Datafile collation '{_header.Pragmas.Collation}' is different from engine settings. Use Rebuild database to change collation."); } _locker = new LockService(_header.Pragmas); _walIndex = new WalIndexService(_disk, _locker); if (_disk.GetFileLength(FileOrigin.Log) > 0) { _walIndex.RestoreIndex(ref _header); } _sortDisk = new SortDisk(_settings.CreateTempFactory(), 819200, _header.Pragmas); _monitor = new TransactionMonitor(_header, _locker, _disk, _walIndex); InitializeSystemCollections(); return true; } catch (Exception ex) { Close(ex); throw; } } internal List Close() { if (_state.Disposed) { return new List(); } _state.Disposed = true; TryCatch tryCatch = new TryCatch(); tryCatch.Catch(delegate { _monitor?.Dispose(); }); HeaderPage header = _header; if (header != null && header.Pragmas.Checkpoint > 0) { tryCatch.Catch(delegate { _walIndex?.TryCheckpoint(); }); } tryCatch.Catch(delegate { _disk?.Dispose(); }); tryCatch.Catch(delegate { _sortDisk?.Dispose(); }); tryCatch.Catch(delegate { _locker?.Dispose(); }); return tryCatch.Exceptions; } internal List Close(Exception ex) { if (_state.Disposed) { return new List(); } _state.Disposed = true; TryCatch tryCatch = new TryCatch(ex); tryCatch.Catch(delegate { _monitor?.Dispose(); }); tryCatch.Catch(delegate { _disk?.Dispose(); }); tryCatch.Catch(delegate { _sortDisk?.Dispose(); }); tryCatch.Catch(delegate { _locker?.Dispose(); }); if (tryCatch.InvalidDatafileState) { tryCatch.Catch(delegate { _disk.MarkAsInvalidState(); }); } return tryCatch.Exceptions; } public int Checkpoint() { return _walIndex.Checkpoint(); } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { Close(); } private void InitializeSystemCollections() { RegisterSystemCollection("$database", () => SysDatabase()); RegisterSystemCollection("$cols", () => SysCols()); RegisterSystemCollection("$indexes", () => SysIndexes()); RegisterSystemCollection("$sequences", () => SysSequences()); RegisterSystemCollection("$transactions", () => SysTransactions()); RegisterSystemCollection("$snapshots", () => SysSnapshots()); RegisterSystemCollection("$open_cursors", () => SysOpenCursors()); RegisterSystemCollection(new SysFile()); RegisterSystemCollection(new SysDump(_header, _monitor)); RegisterSystemCollection(new SysPageList(_header, _monitor)); RegisterSystemCollection(new SysQuery(this)); } private IEnumerable SysCols() { foreach (KeyValuePair collection in _header.GetCollections()) { yield return new BsonDocument { ["name"] = collection.Key, ["type"] = "user" }; } foreach (KeyValuePair systemCollection in _systemCollections) { yield return new BsonDocument { ["name"] = systemCollection.Key, ["type"] = "system" }; } } private IEnumerable SysDatabase() { Version version = typeof(LiteEngine).GetTypeInfo().Assembly.GetName().Version; yield return new BsonDocument { ["name"] = _disk.GetName(FileOrigin.Data), ["encrypted"] = _settings.Password != null, ["readOnly"] = _settings.ReadOnly, ["lastPageID"] = (int)_header.LastPageID, ["freeEmptyPageID"] = (int)_header.FreeEmptyPageList, ["creationTime"] = _header.CreationTime, ["dataFileSize"] = (int)_disk.GetFileLength(FileOrigin.Data), ["logFileSize"] = (int)_disk.GetFileLength(FileOrigin.Log), ["currentReadVersion"] = _walIndex.CurrentReadVersion, ["lastTransactionID"] = _walIndex.LastTransactionID, ["engine"] = $"litedb-ce-v{version.Major}.{version.Minor}.{version.Build}", ["pragmas"] = new BsonDocument(_header.Pragmas.Pragmas.ToDictionary((Pragma x) => x.Name, (Pragma x) => x.Get())), ["cache"] = new BsonDocument { ["extendSegments"] = _disk.Cache.ExtendSegments, ["extendPages"] = _disk.Cache.ExtendPages, ["freePages"] = _disk.Cache.FreePages, ["readablePages"] = _disk.Cache.GetPages().Count, ["writablePages"] = _disk.Cache.WritablePages, ["pagesInUse"] = _disk.Cache.PagesInUse }, ["transactions"] = new BsonDocument { ["open"] = _monitor.Transactions.Count, ["maxOpenTransactions"] = 100, ["initialTransactionSize"] = _monitor.InitialSize, ["availableSize"] = _monitor.FreePages } }; } private IEnumerable SysIndexes() { TransactionService transaction = _monitor.GetThreadTransaction(); foreach (KeyValuePair collection in _header.GetCollections()) { Snapshot snapshot = transaction.CreateSnapshot(LockMode.Read, collection.Key, addIfNotExists: false); foreach (CollectionIndex collectionIndex in snapshot.CollectionPage.GetCollectionIndexes()) { yield return new BsonDocument { ["collection"] = collection.Key, ["name"] = collectionIndex.Name, ["expression"] = collectionIndex.Expression, ["unique"] = collectionIndex.Unique }; } } } private IEnumerable SysOpenCursors() { foreach (TransactionService transaction in _monitor.Transactions) { foreach (CursorInfo openCursor in transaction.OpenCursors) { yield return new BsonDocument { ["threadID"] = transaction.ThreadID, ["transactionID"] = (int)transaction.TransactionID, ["elapsedMS"] = (int)openCursor.Elapsed.ElapsedMilliseconds, ["collection"] = openCursor.Collection, ["mode"] = (openCursor.Query.ForUpdate ? "write" : "read"), ["sql"] = openCursor.Query.ToSQL(openCursor.Collection).Replace(Environment.NewLine, " "), ["running"] = openCursor.Elapsed.IsRunning, ["fetched"] = openCursor.Fetched }; } } } private IEnumerable SysSequences() { KeyValuePair[] array = _sequences.ToArray(); KeyValuePair[] array2 = array; for (int i = 0; i < array2.Length; i++) { KeyValuePair keyValuePair = array2[i]; yield return new BsonDocument { ["collection"] = keyValuePair.Key, ["value"] = keyValuePair.Value }; } } private IEnumerable SysSnapshots() { foreach (TransactionService transaction in _monitor.Transactions) { foreach (Snapshot snapshot in transaction.Snapshots) { yield return new BsonDocument { ["transactionID"] = (int)transaction.TransactionID, ["collection"] = snapshot.CollectionName, ["mode"] = snapshot.Mode.ToString(), ["readVersion"] = snapshot.ReadVersion, ["pagesInMemory"] = snapshot.LocalPages.Count, ["collectionDirty"] = snapshot.CollectionPage?.IsDirty ?? false }; } } } private IEnumerable SysTransactions() { foreach (TransactionService transaction in _monitor.Transactions) { yield return new BsonDocument { ["threadID"] = transaction.ThreadID, ["transactionID"] = (int)transaction.TransactionID, ["startTime"] = transaction.StartTime, ["mode"] = transaction.Mode.ToString(), ["transactionSize"] = transaction.Pages.TransactionSize, ["maxTransactionSize"] = transaction.MaxTransactionSize, ["pagesInLogFile"] = transaction.Pages.DirtyPages.Count, ["newPages"] = transaction.Pages.NewPages.Count, ["deletedPages"] = transaction.Pages.DeletedPages, ["modifiedPages"] = transaction.Snapshots.Select((Snapshot x) => x.GetWritablePages(dirty: true, includeCollectionPage: true).Count()).Sum() }; } } } internal class FileReaderError { public DateTime Created { get; } = DateTime.Now; public FileOrigin Origin { get; set; } public long Position { get; set; } public uint? PageID { get; set; } public PageType PageType { get; set; } public string Collection { get; set; } public string Message { get; set; } public Exception Exception { get; set; } } internal class FileReaderV7 : IFileReader, IDisposable { private const int V7_PAGE_SIZE = 4096; private readonly EngineSettings _settings; private Stream _stream; private AesEncryption _aes; private BsonDocument _header; private byte[] _buffer = new byte[4096]; private bool _disposedValue; private static readonly byte[] arrayByteEmpty = new byte[0]; public IDictionary GetPragmas() { return new Dictionary { { "USER_VERSION", _header["userVersion"].AsInt32 }, { "CHECKPOINT", 1000 }, { "TIMEOUT", 60 }, { "LIMIT_SIZE", long.MaxValue }, { "UTC_DATE", true } }; } public FileReaderV7(EngineSettings settings) { _settings = settings; } public void Open() { IStreamFactory streamFactory = _settings.CreateDataFactory(useAesStream: false); _stream = streamFactory.GetStream(canWrite: true, sequencial: true); _header = ReadPage(0u); if (_settings.Password == null && !_header["salt"].AsBinary.IsFullZero()) { throw LiteException.InvalidPassword(); } if (_settings.Password != null) { if (_header["salt"].AsBinary.IsFullZero()) { throw LiteException.FileNotEncrypted(); } if (!AesEncryption.HashSHA1(_settings.Password).SequenceEqual(_header["password"].AsBinary)) { throw LiteException.InvalidPassword(); } } _aes = ((_settings.Password == null) ? null : new AesEncryption(_settings.Password, _header["salt"].AsBinary)); } public static bool IsVersion(byte[] buffer) { string @string = Encoding.UTF8.GetString(buffer, 25, "** This is a LiteDB file **".Length); byte b = buffer[52]; if (@string == "** This is a LiteDB file **") { return b == 7; } return false; } public IEnumerable GetCollections() { return _header["collections"].AsDocument.Keys; } public IEnumerable GetIndexes(string collection) { uint asInt = (uint)_header["collections"].AsDocument[collection].AsInt32; BsonDocument bsonDocument = ReadPage(asInt); foreach (BsonValue item in bsonDocument["indexes"].AsArray) { string text = Regex.Replace(item["name"].AsString, "[^a-z0-9]", "", RegexOptions.IgnoreCase | RegexOptions.Compiled); if (text.Length > Constants.INDEX_NAME_MAX_LENGTH) { text = text.Substring(0, Constants.INDEX_NAME_MAX_LENGTH); } yield return new IndexInfo { Collection = collection, Name = text, Expression = item["expression"].AsString, Unique = item["unique"].AsBoolean }; } } public IEnumerable GetDocuments(string collection) { uint asInt = (uint)_header["collections"].AsDocument[collection].AsInt32; uint asInt2 = (uint)ReadPage(asInt)["indexes"][0]["headPageID"].AsInt32; HashSet hashSet = VisitIndexPages(asInt2); foreach (uint item in hashSet) { BsonDocument bsonDocument = ReadPage(item); foreach (BsonValue item2 in bsonDocument["nodes"].AsArray) { BsonValue dataBlock = item2["dataBlock"]; if (dataBlock["pageID"].AsInt32 == -1) { continue; } BsonDocument bsonDocument2 = ReadPage((uint)dataBlock["pageID"].AsInt32); if (bsonDocument2["pageType"].AsInt32 != 4) { continue; } BsonDocument bsonDocument3 = bsonDocument2["blocks"].AsArray.FirstOrDefault((BsonValue x) => x["index"] == dataBlock["index"])?.AsDocument; if (bsonDocument3 == null) { continue; } byte[] array = ((bsonDocument3["extendPageID"] == -1) ? bsonDocument3["data"].AsBinary : ReadExtendData((uint)bsonDocument3["extendPageID"].AsInt32)); if (array.Length == 0) { continue; } BsonDocument bsonDocument4 = BsonSerializer.Deserialize(array); if (collection == "_chunks") { string[] array2 = bsonDocument4["_id"].AsString.Split(new char[1] { '\\' }); if (!int.TryParse(array2[1], out var result)) { throw LiteException.InvalidFormat("_id"); } bsonDocument4["_id"] = new BsonDocument { ["f"] = array2[0], ["n"] = result }; } yield return bsonDocument4; } } } private BsonDocument ReadPage(uint pageID) { if (pageID * 4096 > _stream.Length) { return null; } _stream.Position = pageID * 4096; _stream.Read(_buffer, 0, 4096); if (_aes != null && pageID != 0) { _buffer = _aes.Decrypt(_buffer); } ByteReader byteReader = new ByteReader(_buffer); BsonDocument bsonDocument = new BsonDocument { ["pageID"] = (int)byteReader.ReadUInt32(), ["pageType"] = (int)byteReader.ReadByte(), ["prevPageID"] = (int)byteReader.ReadUInt32(), ["nextPageID"] = (int)byteReader.ReadUInt32(), ["itemCount"] = (int)byteReader.ReadUInt16() }; byteReader.ReadBytes(10); if (bsonDocument["pageType"] == 1) { string strA = byteReader.ReadString(27); byte b = byteReader.ReadByte(); if (string.CompareOrdinal(strA, "** This is a LiteDB file **") != 0 || b != 7) { throw LiteException.InvalidDatabase(); } byteReader.ReadBytes(10); bsonDocument["userVersion"] = (int)byteReader.ReadUInt16(); bsonDocument["password"] = byteReader.ReadBytes(20); bsonDocument["salt"] = byteReader.ReadBytes(16); bsonDocument["collections"] = new BsonDocument(); byte b2 = byteReader.ReadByte(); for (int i = 0; i < b2; i++) { string name = byteReader.ReadString(); uint num = byteReader.ReadUInt32(); bsonDocument["collections"][name] = (int)num; } } else if (bsonDocument["pageType"] == 2) { bsonDocument["collectionName"] = byteReader.ReadString(); bsonDocument["indexes"] = new BsonArray(); byteReader.ReadBytes(12); for (int j = 0; j < 16; j++) { BsonDocument bsonDocument2 = new BsonDocument(); string text = byteReader.ReadString(); int num2 = text.IndexOf('='); if (num2 > 0) { bsonDocument2["name"] = text.Substring(0, num2); bsonDocument2["expression"] = text.Substring(num2 + 1); } else { bsonDocument2["name"] = text; bsonDocument2["expression"] = "$." + text; } bsonDocument2["unique"] = byteReader.ReadBoolean(); bsonDocument2["headPageID"] = (int)byteReader.ReadUInt32(); byteReader.ReadBytes(12); if (text.Length > 0) { bsonDocument["indexes"].AsArray.Add(bsonDocument2); } } } else if (bsonDocument["pageType"] == 3) { bsonDocument["nodes"] = new BsonArray(); for (int k = 0; k < bsonDocument["itemCount"].AsInt32; k++) { BsonDocument bsonDocument3 = new BsonDocument { ["index"] = (int)byteReader.ReadUInt16() }; byte b3 = byteReader.ReadByte(); byteReader.ReadBytes(13); ushort num3 = byteReader.ReadUInt16(); byteReader.ReadBytes(1 + num3); bsonDocument3["dataBlock"] = new BsonDocument { ["pageID"] = (int)byteReader.ReadUInt32(), ["index"] = (int)byteReader.ReadUInt16() }; bsonDocument3["prev"] = new BsonDocument { ["pageID"] = (int)byteReader.ReadUInt32(), ["index"] = (int)byteReader.ReadUInt16() }; bsonDocument3["next"] = new BsonDocument { ["pageID"] = (int)byteReader.ReadUInt32(), ["index"] = (int)byteReader.ReadUInt16() }; byteReader.ReadBytes((b3 - 1) * 12); bsonDocument["nodes"].AsArray.Add(bsonDocument3); } } else if (bsonDocument["pageType"] == 4) { bsonDocument["blocks"] = new BsonArray(); for (int l = 0; l < bsonDocument["itemCount"].AsInt32; l++) { BsonDocument bsonDocument4 = new BsonDocument { ["index"] = (int)byteReader.ReadUInt16(), ["extendPageID"] = (int)byteReader.ReadUInt32() }; ushort count = byteReader.ReadUInt16(); bsonDocument4["data"] = byteReader.ReadBytes(count); bsonDocument["blocks"].AsArray.Add(bsonDocument4); } } else if (bsonDocument["pageType"] == 5) { bsonDocument["data"] = byteReader.ReadBytes(bsonDocument["itemCount"].AsInt32); } return bsonDocument; } private byte[] ReadExtendData(uint extendPageID) { using MemoryStream memoryStream = new MemoryStream(); while (extendPageID != uint.MaxValue) { BsonDocument bsonDocument = ReadPage(extendPageID); if (bsonDocument["pageType"].AsInt32 != 5) { return arrayByteEmpty; } memoryStream.Write(bsonDocument["data"].AsBinary, 0, bsonDocument["itemCount"].AsInt32); extendPageID = (uint)bsonDocument["nextPageID"].AsInt32; } return memoryStream.ToArray(); } private HashSet VisitIndexPages(uint startPageID) { HashSet hashSet = new HashSet { startPageID }; HashSet hashSet2 = new HashSet(); while (hashSet.Count > 0) { uint num = hashSet.First(); hashSet.Remove(num); BsonDocument bsonDocument = ReadPage(num); if (bsonDocument == null || bsonDocument["pageType"] != 3) { continue; } hashSet2.Add(num); foreach (BsonValue item in bsonDocument["nodes"].AsArray) { uint asInt = (uint)item["prev"]["pageID"].AsInt32; uint asInt2 = (uint)item["next"]["pageID"].AsInt32; if (!hashSet2.Contains(asInt)) { hashSet.Add(asInt); } if (!hashSet2.Contains(asInt2)) { hashSet.Add(asInt2); } } } return hashSet2; } protected virtual void Dispose(bool disposing) { if (!_disposedValue) { if (disposing) { _stream?.Dispose(); _aes?.Dispose(); } _disposedValue = true; } } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } } internal class FileReaderV8 : IFileReader, IDisposable { private struct PageInfo { public uint PageID; public FileOrigin Origin; public PageType PageType; public long Position; public uint ColID; } private readonly Dictionary _collections = new Dictionary(); private readonly Dictionary> _indexes = new Dictionary>(); private readonly Dictionary> _collectionsDataPages = new Dictionary>(); private readonly Dictionary _pragmas = new Dictionary { ["USER_VERSION"] = 0, ["CHECKPOINT"] = 1000, ["TIMEOUT"] = 60, ["UTC_DATE"] = true, ["LIMIT_SIZE"] = long.MaxValue }; private Stream _dataStream; private Stream _logStream; private readonly IDictionary _logIndexMap = new Dictionary(); private uint _maxPageID; private bool _disposed; private readonly EngineSettings _settings; private readonly IList _errors; public FileReaderV8(EngineSettings settings, IList errors) { _settings = settings; _errors = errors; } public void Open() { try { IStreamFactory streamFactory = _settings.CreateDataFactory(); IStreamFactory streamFactory2 = _settings.CreateLogFactory(); _maxPageID = (uint)((streamFactory.GetLength() + streamFactory2.GetLength()) / 8192); _dataStream = streamFactory.GetStream(canWrite: true, sequencial: false); _dataStream.Position = 0L; if (streamFactory2.Exists()) { _logStream = streamFactory2.GetStream(canWrite: false, sequencial: true); LoadIndexMap(); } LoadPragmas(); LoadDataPages(); LoadCollections(); LoadIndexes(); } catch (Exception ex) { HandleError(ex, default(PageInfo)); } } public IDictionary GetPragmas() { return _pragmas; } public IEnumerable GetCollections() { return _collections.Keys; } public IEnumerable GetIndexes(string collection) { if (!_indexes.ContainsKey(collection)) { return new List(); } return _indexes[collection]; } public IEnumerable GetDocuments(string collection) { if (!_collections.ContainsKey(collection)) { yield break; } uint colID = _collections[collection]; if (!_collectionsDataPages.ContainsKey(colID)) { yield break; } List list = _collectionsDataPages[colID]; HashSet uniqueIDs = new HashSet(); foreach (uint item in list) { PageInfo pageInfo; Result result = ReadPage(item, out pageInfo); if (result.Fail) { HandleError(result.Exception, pageInfo); continue; } PageBuffer buffer = result.Value.Buffer; byte itemsCount = result.Value.ItemsCount; byte highestIndex = result.Value.HighestIndex; if (itemsCount == 0 || highestIndex == byte.MaxValue) { continue; } for (int i = 0; i <= highestIndex; i++) { BsonDocument bsonDocument; try { int offset = BasePage.CalcPositionAddr((byte)i); int offset2 = BasePage.CalcLengthAddr((byte)i); ushort num = buffer.ReadUInt16(offset); ushort num2 = buffer.ReadUInt16(offset2); if (num == 0) { continue; } Constants.ENSURE(num > 0 && num2 > 0, "Invalid footer ref position {0} with length {1}", num, num2); Constants.ENSURE(num + num2 < 8192, "Invalid footer ref position {0} with length {1}", num, num2); BufferSlice bufferSlice = buffer.Slice(num, num2); bool num3 = bufferSlice.ReadBool(0); PageAddress pageAddress = bufferSlice.ReadPageAddress(1); BufferSlice bufferSlice2 = bufferSlice.Slice(6, bufferSlice.Count - 6); if (num3) { continue; } using (MemoryStream memoryStream = new MemoryStream()) { memoryStream.Write(bufferSlice2.Array, bufferSlice2.Offset, bufferSlice2.Count); while (!pageAddress.IsEmpty) { Result result2 = ReadPage(pageAddress.PageID, out pageInfo); if (result2.Fail) { throw result2.Exception; } PageBuffer buffer2 = result2.Value.Buffer; Constants.ENSURE(result2.Value.PageType == PageType.Data, "Invalid PageType (excepted Data, get {0})", result2.Value.PageType); Constants.ENSURE(result2.Value.ColID == colID, "Invalid ColID in this page (expected {0}, get {1})", colID, result2.Value.ColID); Constants.ENSURE(result2.Value.ItemsCount > 0, "Page with no items count"); offset = BasePage.CalcPositionAddr(pageAddress.Index); offset2 = BasePage.CalcLengthAddr(pageAddress.Index); num = buffer2.ReadUInt16(offset); num2 = buffer2.ReadUInt16(offset2); Constants.ENSURE(num2 > 0, "Last DataBlock request a next extend to {0}, but this block are empty footer", pageAddress); bufferSlice = buffer2.Slice(num, num2); bool conditional = bufferSlice.ReadBool(0); pageAddress = bufferSlice.ReadPageAddress(1); bufferSlice2 = bufferSlice.Slice(6, bufferSlice.Count - 6); Constants.ENSURE(conditional, "Next datablock always be an extend. Invalid data block {0}", pageAddress); memoryStream.Write(bufferSlice2.Array, bufferSlice2.Offset, bufferSlice2.Count); } using BufferReader bufferReader = new BufferReader(memoryStream.ToArray()); Result result3 = bufferReader.ReadDocument(); BsonValue bsonValue = result3.Value["_id"]; Constants.ENSURE(!(bsonValue == BsonValue.Null) && !(bsonValue == BsonValue.MinValue) && !(bsonValue == BsonValue.MaxValue), "Invalid _id value: {0}", bsonValue); Constants.ENSURE(!uniqueIDs.Contains(bsonValue), "Duplicated _id value: {0}", bsonValue); uniqueIDs.Add(bsonValue); if (result3.Fail) { HandleError(result3.Exception, pageInfo); } bsonDocument = result3.Value; } goto IL_04c5; } catch (Exception ex) { HandleError(ex, pageInfo); bsonDocument = null; goto IL_04c5; } IL_04c5: if (bsonDocument != null) { yield return bsonDocument; } } } } private void LoadPragmas() { PageInfo pageInfo; Result result = ReadPage(0u, out pageInfo); if (result.Ok) { PageBuffer buffer = result.Value.Buffer; _pragmas["USER_VERSION"] = buffer.ReadInt32(76); _pragmas["CHECKPOINT"] = buffer.ReadInt32(97); _pragmas["TIMEOUT"] = buffer.ReadInt32(88); _pragmas["UTC_DATE"] = buffer.ReadBool(96); _pragmas["LIMIT_SIZE"] = buffer.ReadInt64(101); } else { HandleError(result.Exception, pageInfo); } } private void LoadDataPages() { PageInfo pageInfo; uint num = ReadPage(0u, out pageInfo).GetValue().Buffer.ReadUInt32(64); Constants.ENSURE(num <= _maxPageID, "LastPageID {0} should be less or equals to maxPageID {1}", num, _maxPageID); for (uint num2 = 0u; num2 <= num; num2++) { Result result = ReadPage(num2, out pageInfo); if (result.Ok) { BasePage value = result.Value; if (value.PageType == PageType.Data) { if (_collectionsDataPages.TryGetValue(value.ColID, out var value2)) { value2.Add(value.PageID); continue; } _collectionsDataPages[value.ColID] = new List { value.PageID }; } } else { HandleError(result.Exception, pageInfo); } } } private void LoadCollections() { PageInfo pageInfo; BufferSlice bufferSlice = ReadPage(0u, out pageInfo).GetValue().Buffer.Slice(192, 8000); using (BufferReader bufferReader = new BufferReader(new BufferSlice[1] { bufferSlice })) { Result result = bufferReader.ReadDocument(); BsonDocument value = result.Value; foreach (string key in value.Keys) { BsonValue bsonValue = value[key]; if (!bsonValue.IsNumber) { HandleError($"ColID expect a number but get {bsonValue}", pageInfo); } else { _collections[key] = (uint)value[key].AsInt32; } } if (result.Fail) { HandleError(result.Exception, pageInfo); } } foreach (KeyValuePair> collectionsDataPage in _collectionsDataPages) { if (!_collections.ContainsValue(collectionsDataPage.Key)) { _collections["col_" + collectionsDataPage.Key] = collectionsDataPage.Key; } } } private void LoadIndexes() { foreach (KeyValuePair collection in _collections) { PageInfo pageInfo; Result result = ReadPage(collection.Value, out pageInfo); if (result.Fail) { HandleError(result.Exception, pageInfo); continue; } PageBuffer buffer = result.Value.Buffer; byte b = buffer.ReadByte(96); int num = 97; try { for (int i = 0; i < b; i++) { num += 2; int length; string text = buffer.ReadCString(num, out length); num += length; int length2; string text2 = buffer.ReadCString(num, out length2); num += length2; bool unique = buffer.ReadBool(num); num++; num += 15; Constants.ENSURE(!string.IsNullOrEmpty(text), "Index name can't be empty (collection {0} - index: {1})", collection.Key, i); Constants.ENSURE(!string.IsNullOrEmpty(text2), "Index expression can't be empty (collection {0} - index: {1})", collection.Key, i); IndexInfo item = new IndexInfo { Collection = collection.Key, Name = text, Expression = text2, Unique = unique }; if (!(text == "_id")) { if (_indexes.TryGetValue(collection.Key, out var value)) { value.Add(item); continue; } _indexes[collection.Key] = new List { item }; } } } catch (Exception ex) { HandleError(ex, pageInfo); } } } public static bool IsVersion(byte[] buffer) { string @string = Encoding.UTF8.GetString(buffer, 32, "** This is a LiteDB file **".Length); byte b = buffer[59]; if (!(@string == "** This is a LiteDB file **") || b != 8) { return buffer[0] == 1; } return true; } private void LoadIndexMap() { PageBuffer pageBuffer = new PageBuffer(new byte[8192], 0, 0); Dictionary> dictionary = new Dictionary>(); List list = new List(); long num = 0L; PageInfo pageInfo = default(PageInfo); pageInfo.Origin = FileOrigin.Log; PageInfo pageInfo2 = pageInfo; _logStream.Position = 0L; while (_logStream.Position < _logStream.Length) { try { _logStream.Position = (pageInfo2.Position = num); int num2 = _logStream.Read(pageBuffer.Array, pageBuffer.Offset, 8192); if (pageBuffer.IsBlank()) { num += 8192; continue; } uint pageID = pageBuffer.ReadUInt32(0); bool flag = pageBuffer.ReadBool(18); uint num3 = pageBuffer.ReadUInt32(14); pageInfo2.PageID = pageID; pageInfo2.ColID = pageBuffer.ReadUInt32(19); Constants.ENSURE(num2 == 8192, "Page position {0} read only than {1} bytes (instead {2})", _logStream, num2, 8192); PagePosition item = new PagePosition(pageID, num); if (dictionary.TryGetValue(num3, out var value)) { value.Add(item); } else { dictionary[num3] = new List { item }; } if (flag) { list.Add(num3); } } catch (Exception ex) { HandleError(ex, pageInfo2); } finally { num += 8192; } } foreach (uint item2 in list) { foreach (PagePosition item3 in dictionary[item2]) { _logIndexMap[item3.PageID] = item3.Position; } } } private Result ReadPage(uint pageID, out PageInfo pageInfo) { pageInfo = new PageInfo { PageID = pageID }; try { Constants.ENSURE(pageID <= _maxPageID, "PageID: {0} should be less then or equals to maxPageID: {1}", pageID, _maxPageID); PageBuffer pageBuffer = new PageBuffer(new byte[8192], 0, 8192); Stream stream; if (_logIndexMap.TryGetValue(pageID, out pageInfo.Position)) { pageInfo.Origin = FileOrigin.Log; stream = _logStream; } else { pageInfo.Origin = FileOrigin.Data; pageInfo.Position = BasePage.GetPagePosition(pageID); stream = _dataStream; } stream.Position = pageInfo.Position; int num = stream.Read(pageBuffer.Array, pageBuffer.Offset, pageBuffer.Count); Constants.ENSURE(num == 8192, "Page position {0} read only than {1} bytes (instead {2})", stream.Position, num, 8192); BasePage basePage = new BasePage(pageBuffer); pageInfo.ColID = basePage.ColID; Constants.ENSURE(basePage.PageID == pageID, "Expect read pageID: {0} but header contains pageID: {1}", pageID, basePage.PageID); return basePage; } catch (Exception ex) { return new Result(null, ex); } } private void HandleError(Exception ex, PageInfo pageInfo) { string key = _collections.FirstOrDefault((KeyValuePair x) => x.Value == pageInfo.ColID).Key; _errors.Add(new FileReaderError { Position = pageInfo.Position, Origin = pageInfo.Origin, PageID = pageInfo.PageID, PageType = pageInfo.PageType, Collection = key, Message = ex.Message, Exception = ex }); if (ex is IOException) { throw ex; } } private void HandleError(string message, PageInfo pageInfo) { HandleError(new Exception(message), pageInfo); } protected virtual void Dispose(bool disposing) { if (!_disposed) { _dataStream?.Dispose(); _logStream?.Dispose(); _disposed = true; } } public void Dispose() { Dispose(disposing: true); GC.SuppressFinalize(this); } } internal interface IFileReader : IDisposable { void Open(); IDictionary GetPragmas(); IEnumerable GetCollections(); IEnumerable GetIndexes(string name); IEnumerable GetDocuments(string collection); } internal class IndexInfo { public string Collection { get; set; } public string Name { get; set; } public string Expression { get; set; } public bool Unique { get; set; } } public interface ILiteEngine : IDisposable { int Checkpoint(); long Rebuild(RebuildOptions options); bool BeginTrans(); bool Commit(); bool Rollback(); IBsonDataReader Query(string collection, Query query); int Insert(string collection, IEnumerable docs, BsonAutoId autoId); int Update(string collection, IEnumerable docs); int UpdateMany(string collection, BsonExpression transform, BsonExpression predicate); int Upsert(string collection, IEnumerable docs, BsonAutoId autoId); int Delete(string collection, IEnumerable ids); int DeleteMany(string collection, BsonExpression predicate); bool DropCollection(string name); bool RenameCollection(string name, string newName); bool EnsureIndex(string collection, string name, BsonExpression expression, bool unique); bool DropIndex(string collection, string name); BsonValue Pragma(string name); bool Pragma(string name, BsonValue value); } internal enum PageType { Empty, Header, Collection, Index, Data } internal class BasePage { protected readonly PageBuffer _buffer; public const int SLOT_SIZE = 4; public const int P_PAGE_ID = 0; public const int P_PAGE_TYPE = 4; public const int P_PREV_PAGE_ID = 5; public const int P_NEXT_PAGE_ID = 9; public const int P_INITIAL_SLOT = 13; public const int P_TRANSACTION_ID = 14; public const int P_IS_CONFIRMED = 18; public const int P_COL_ID = 19; public const int P_ITEMS_COUNT = 23; public const int P_USED_BYTES = 24; public const int P_FRAGMENTED_BYTES = 26; public const int P_NEXT_FREE_POSITION = 28; public const int P_HIGHEST_INDEX = 30; private byte _startIndex; public uint PageID { get; } public PageType PageType { get; private set; } public uint PrevPageID { get; set; } public uint NextPageID { get; set; } public byte PageListSlot { get; set; } public byte ItemsCount { get; private set; } public ushort UsedBytes { get; private set; } public ushort FragmentedBytes { get; private set; } public ushort NextFreePosition { get; private set; } public byte HighestIndex { get; private set; } public int FreeBytes { get { if (ItemsCount != byte.MaxValue) { return 8160 - UsedBytes - FooterSize; } return 0; } } public int FooterSize { get { if (HighestIndex != byte.MaxValue) { return (HighestIndex + 1) * 4; } return 0; } } public uint ColID { get; set; } public uint TransactionID { get; set; } public bool IsConfirmed { get; set; } public bool IsDirty { get; set; } public PageBuffer Buffer => _buffer; public BasePage(PageBuffer buffer, uint pageID, PageType pageType) { _buffer = buffer; PageID = pageID; PageType = pageType; PrevPageID = uint.MaxValue; NextPageID = uint.MaxValue; PageListSlot = byte.MaxValue; ColID = uint.MaxValue; TransactionID = uint.MaxValue; IsConfirmed = false; ItemsCount = 0; UsedBytes = 0; FragmentedBytes = 0; NextFreePosition = 32; HighestIndex = byte.MaxValue; IsDirty = false; _buffer.Write(PageID, 0); _buffer.Write((byte)PageType, 4); } public BasePage(PageBuffer buffer) { _buffer = buffer; PageID = _buffer.ReadUInt32(0); PageType = (PageType)_buffer.ReadByte(4); PrevPageID = _buffer.ReadUInt32(5); NextPageID = _buffer.ReadUInt32(9); PageListSlot = _buffer.ReadByte(13); TransactionID = _buffer.ReadUInt32(14); IsConfirmed = _buffer.ReadBool(18); ColID = _buffer.ReadUInt32(19); ItemsCount = _buffer.ReadByte(23); UsedBytes = _buffer.ReadUInt16(24); FragmentedBytes = _buffer.ReadUInt16(26); NextFreePosition = _buffer.ReadUInt16(28); HighestIndex = _buffer.ReadByte(30); } public virtual PageBuffer UpdateBuffer() { Constants.ENSURE(PageID == _buffer.ReadUInt32(0), "pageID can't be changed"); _buffer.Write(PrevPageID, 5); _buffer.Write(NextPageID, 9); _buffer.Write(PageListSlot, 13); _buffer.Write(TransactionID, 14); _buffer.Write(IsConfirmed, 18); _buffer.Write(ColID, 19); _buffer.Write(ItemsCount, 23); _buffer.Write(UsedBytes, 24); _buffer.Write(FragmentedBytes, 26); _buffer.Write(NextFreePosition, 28); _buffer.Write(HighestIndex, 30); return _buffer; } public void MarkAsEmtpy() { IsDirty = true; PageType = PageType.Empty; PrevPageID = uint.MaxValue; NextPageID = uint.MaxValue; PageListSlot = byte.MaxValue; ColID = uint.MaxValue; TransactionID = uint.MaxValue; IsConfirmed = false; ItemsCount = 0; UsedBytes = 0; FragmentedBytes = 0; NextFreePosition = 32; HighestIndex = byte.MaxValue; _buffer.Clear(32, 8160); _buffer.Write((byte)PageType, 4); } public BufferSlice Get(byte index) { Constants.ENSURE(ItemsCount > 0, "should have items in this page"); Constants.ENSURE(HighestIndex != byte.MaxValue, "should have at least 1 index in this page"); Constants.ENSURE(index <= HighestIndex, "get only index below highest index"); int offset = CalcPositionAddr(index); int offset2 = CalcLengthAddr(index); ushort num = _buffer.ReadUInt16(offset); ushort num2 = _buffer.ReadUInt16(offset2); Constants.ENSURE(IsValidPos(num), "invalid segment position in index footer: {0}/{1}", this, index); Constants.ENSURE(IsValidLen(num2), "invalid segment length in index footer: {0}/{1}", this, index); return _buffer.Slice(num, num2); } public BufferSlice Insert(ushort bytesLength, out byte index) { index = byte.MaxValue; return InternalInsert(bytesLength, ref index); } private BufferSlice InternalInsert(ushort bytesLength, ref byte index) { bool flag = index == byte.MaxValue; Constants.ENSURE(_buffer.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable to support changes"); Constants.ENSURE(bytesLength > 0, "must insert more than 0 bytes"); Constants.ENSURE(FreeBytes >= bytesLength + (flag ? 4 : 0), "length must be always lower than current free space"); Constants.ENSURE(ItemsCount < byte.MaxValue, "page full"); Constants.ENSURE(FreeBytes >= FragmentedBytes, "fragmented bytes must be at most free bytes"); if (FreeBytes < bytesLength + (flag ? 4 : 0)) { throw LiteException.InvalidFreeSpacePage(PageID, FreeBytes, bytesLength + (flag ? 4 : 0)); } int num = FreeBytes - FragmentedBytes - (flag ? 4 : 0); Constants.ENSURE(num == 8192 - NextFreePosition - FooterSize - (flag ? 4 : 0), "continuousBlock must be same as from NextFreePosition"); if (bytesLength > num) { Defrag(); } if (index == byte.MaxValue) { index = GetFreeIndex(); } if (index > HighestIndex || HighestIndex == byte.MaxValue) { Constants.ENSURE(index == (byte)(HighestIndex + 1), "new index must be next highest index"); HighestIndex = index; } int offset = CalcPositionAddr(index); int offset2 = CalcLengthAddr(index); Constants.ENSURE(_buffer.ReadUInt16(offset) == 0, "slot position must be empty before use"); Constants.ENSURE(_buffer.ReadUInt16(offset2) == 0, "slot length must be empty before use"); ushort nextFreePosition = NextFreePosition; _buffer.Write(nextFreePosition, offset); _buffer.Write(bytesLength, offset2); ItemsCount++; UsedBytes += bytesLength; NextFreePosition += bytesLength; IsDirty = true; Constants.ENSURE(nextFreePosition + bytesLength <= 8192 - (HighestIndex + 1) * 4, "new buffer slice could not override footer area"); return _buffer.Slice(nextFreePosition, bytesLength); } public void Delete(byte index) { Constants.ENSURE(_buffer.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable to support changes"); int offset = CalcPositionAddr(index); int offset2 = CalcLengthAddr(index); ushort num = _buffer.ReadUInt16(offset); ushort num2 = _buffer.ReadUInt16(offset2); Constants.ENSURE(IsValidPos(num), "invalid segment position"); Constants.ENSURE(IsValidLen(num2), "invalid segment length"); ((BufferSlice)_buffer).Write((ushort)0, offset); ((BufferSlice)_buffer).Write((ushort)0, offset2); ItemsCount--; UsedBytes -= num2; _buffer.Array.Fill(0, _buffer.Offset + num, num2); if (num + num2 == NextFreePosition) { NextFreePosition = num; } else { FragmentedBytes += num2; } if (HighestIndex == index) { UpdateHighestIndex(); } _startIndex = 0; if (ItemsCount == 0) { Constants.ENSURE(HighestIndex == byte.MaxValue, "if there is no items, HighestIndex must be clear"); Constants.ENSURE(UsedBytes == 0, "should be no bytes used in clean page"); NextFreePosition = 32; FragmentedBytes = 0; } IsDirty = true; } public BufferSlice Update(byte index, ushort bytesLength) { Constants.ENSURE(_buffer.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable to support changes"); Constants.ENSURE(bytesLength > 0, "must update more than 0 bytes"); int offset = CalcPositionAddr(index); int offset2 = CalcLengthAddr(index); ushort num = _buffer.ReadUInt16(offset); ushort num2 = _buffer.ReadUInt16(offset2); Constants.ENSURE(IsValidPos(num), "invalid segment position"); Constants.ENSURE(IsValidLen(num2), "invalid segment length"); bool flag = num + num2 == NextFreePosition; IsDirty = true; if (bytesLength == num2) { return _buffer.Slice(num, num2); } if (bytesLength < num2) { ushort num3 = (ushort)(num2 - bytesLength); if (flag) { NextFreePosition -= num3; } else { FragmentedBytes += num3; } UsedBytes -= num3; _buffer.Write(bytesLength, offset2); _buffer.Clear(num + bytesLength, num3); return _buffer.Slice(num, bytesLength); } _buffer.Clear(num, num2); ItemsCount--; UsedBytes -= num2; if (flag) { NextFreePosition = num; } else { FragmentedBytes += num2; } ((BufferSlice)_buffer).Write((ushort)0, offset); ((BufferSlice)_buffer).Write((ushort)0, offset2); return InternalInsert(bytesLength, ref index); } public void Defrag() { Constants.ENSURE(FragmentedBytes > 0, "do not call this when page has no fragmentation"); Constants.ENSURE(_buffer.ShareCounter == Constants.BUFFER_WRITABLE, "page must be writable to support changes"); Constants.ENSURE(HighestIndex < byte.MaxValue, "there is no items in this page to run defrag"); SortedList sortedList = new SortedList(); for (int i = 0; i <= HighestIndex; i++) { int offset = CalcPositionAddr((byte)i); ushort num = _buffer.ReadUInt16(offset); if (num != 0) { Constants.ENSURE(IsValidPos(num), "invalid segment position"); sortedList.Add(num, (byte)i); } } ushort num2 = 32; foreach (KeyValuePair item in sortedList) { byte value = item.Value; ushort key = item.Key; int offset2 = CalcLengthAddr(value); ushort num3 = _buffer.ReadUInt16(offset2); Constants.ENSURE(IsValidLen(num3), "invalid segment length"); if (key != num2) { Constants.ENSURE(key > num2, "current segment position must be greater than current empty space"); System.Buffer.BlockCopy(_buffer.Array, _buffer.Offset + key, _buffer.Array, _buffer.Offset + num2, num3); int offset3 = CalcPositionAddr(value); _buffer.Write(num2, offset3); } num2 += num3; } int count = 8192 - num2 - FooterSize; _buffer.Array.Fill(0, _buffer.Offset + num2, count); FragmentedBytes = 0; NextFreePosition = num2; } private byte GetFreeIndex() { for (byte b = _startIndex; b <= HighestIndex; b++) { int offset = CalcPositionAddr(b); if (_buffer.ReadUInt16(offset) == 0) { _startIndex = (byte)(b + 1); return b; } } return (byte)(HighestIndex + 1); } public IEnumerable GetUsedIndexs() { if (ItemsCount == 0) { yield break; } Constants.ENSURE(HighestIndex != byte.MaxValue, "if has items count, Highest index should be not empty"); for (byte index = 0; index <= HighestIndex; index++) { int offset = CalcPositionAddr(index); if (_buffer.ReadUInt16(offset) != 0) { yield return index; } } } private void UpdateHighestIndex() { Constants.ENSURE(HighestIndex < byte.MaxValue, "can run only if contains a valid HighestIndex"); if (HighestIndex == 0) { HighestIndex = byte.MaxValue; return; } for (int num = HighestIndex - 1; num >= 0; num--) { int offset = CalcPositionAddr((byte)num); ushort num2 = _buffer.ReadUInt16(offset); if (num2 != 0) { Constants.ENSURE(IsValidPos(num2), "invalid segment position"); HighestIndex = (byte)num; return; } } HighestIndex = byte.MaxValue; } private bool IsValidPos(ushort position) { if (position >= 32) { return position < 8192 - FooterSize; } return false; } private bool IsValidLen(ushort length) { if (length > 0) { return length <= 8160 - FooterSize; } return false; } public static int CalcPositionAddr(byte index) { return 8192 - (index + 1) * 4 + 2; } public static int CalcLengthAddr(byte index) { return 8192 - (index + 1) * 4; } public static long GetPagePosition(uint pageID) { checked { return unchecked((long)pageID) * 8192L; } } public static long GetPagePosition(int pageID) { Constants.ENSURE(pageID >= 0, "page could not be less than 0."); return GetPagePosition((uint)pageID); } public static T ReadPage(PageBuffer buffer) where T : BasePage { if (typeof(T) == typeof(BasePage)) { return (T)new BasePage(buffer); } if (typeof(T) == typeof(HeaderPage)) { return (T)(BasePage)new HeaderPage(buffer); } if (typeof(T) == typeof(CollectionPage)) { return (T)(BasePage)new CollectionPage(buffer); } if (typeof(T) == typeof(IndexPage)) { return (T)(BasePage)new IndexPage(buffer); } if (typeof(T) == typeof(DataPage)) { return (T)(BasePage)new DataPage(buffer); } throw new InvalidCastException(); } public static T CreatePage(PageBuffer buffer, uint pageID) where T : BasePage { if (typeof(T) == typeof(CollectionPage)) { return (T)(BasePage)new CollectionPage(buffer, pageID); } if (typeof(T) == typeof(IndexPage)) { return (T)(BasePage)new IndexPage(buffer, pageID); } if (typeof(T) == typeof(DataPage)) { return (T)(BasePage)new DataPage(buffer, pageID); } throw new InvalidCastException(); } public override string ToString() { return $"PageID: {PageID.ToString().PadLeft(4, '0')} : {PageType} ({ItemsCount} Items)"; } } internal class CollectionPage : BasePage { public const int P_INDEXES = 96; public const int P_INDEXES_COUNT = 8096; private readonly Dictionary _indexes = new Dictionary(); public uint[] FreeDataPageList { get; } = new uint[5]; public CollectionIndex PK => _indexes["_id"]; public CollectionPage(PageBuffer buffer, uint pageID) : base(buffer, pageID, PageType.Collection) { for (int i = 0; i < 5; i++) { FreeDataPageList[i] = uint.MaxValue; } } public CollectionPage(PageBuffer buffer) : base(buffer) { Constants.ENSURE(base.PageType == PageType.Collection, "page type must be collection page"); if (base.PageType != PageType.Collection) { throw LiteException.InvalidPageType(PageType.Collection, this); } BufferSlice bufferSlice = _buffer.Slice(32, 8160); using BufferReader bufferReader = new BufferReader(new BufferSlice[1] { bufferSlice }); for (int i = 0; i < 5; i++) { FreeDataPageList[i] = bufferReader.ReadUInt32(); } bufferReader.Skip(64 - bufferReader.Position); byte b = bufferReader.ReadByte(); for (int j = 0; j < b; j++) { CollectionIndex collectionIndex = new CollectionIndex(bufferReader); _indexes[collectionIndex.Name] = collectionIndex; } } public override PageBuffer UpdateBuffer() { if (base.PageType == PageType.Empty) { return base.UpdateBuffer(); } using (BufferWriter bufferWriter = new BufferWriter(_buffer.Slice(32, 8160))) { for (int i = 0; i < 5; i++) { bufferWriter.Write(FreeDataPageList[i]); } bufferWriter.Skip(64 - bufferWriter.Position); bufferWriter.Write((byte)_indexes.Count); foreach (CollectionIndex value in _indexes.Values) { value.UpdateBuffer(bufferWriter); } } return base.UpdateBuffer(); } public CollectionIndex GetCollectionIndex(string name) { if (_indexes.TryGetValue(name, out var value)) { return value; } return null; } public ICollection GetCollectionIndexes() { return _indexes.Values; } public CollectionIndex[] GetCollectionIndexesSlots() { CollectionIndex[] array = new CollectionIndex[_indexes.Max((KeyValuePair x) => x.Value.Slot) + 1]; foreach (CollectionIndex value in _indexes.Values) { array[value.Slot] = value; } return array; } public CollectionIndex InsertCollectionIndex(string name, string expr, bool unique) { int num = 1 + _indexes.Sum((KeyValuePair x) => CollectionIndex.GetLength(x.Value)) + CollectionIndex.GetLength(name, expr); if (_indexes.Count == 255 || num >= 8096) { throw new LiteException(0, "This collection has no more space for new indexes"); } CollectionIndex collectionIndex = new CollectionIndex((byte)((_indexes.Count != 0) ? ((uint)(_indexes.Max((KeyValuePair x) => x.Value.Slot) + 1)) : 0u), 0, name, expr, unique); _indexes[name] = collectionIndex; base.IsDirty = true; return collectionIndex; } public CollectionIndex UpdateCollectionIndex(string name) { base.IsDirty = true; return _indexes[name]; } public void DeleteCollectionIndex(string name) { _indexes.Remove(name); base.IsDirty = true; } } internal class DataPage : BasePage { private static readonly int[] _freePageSlots = new int[4] { 7344, 6120, 4896, 2448 }; public DataPage(PageBuffer buffer) : base(buffer) { Constants.ENSURE(base.PageType == PageType.Data, "Page type must be data page: {0}", base.PageType); if (base.PageType != PageType.Data) { throw LiteException.InvalidPageType(PageType.Data, this); } } public DataPage(PageBuffer buffer, uint pageID) : base(buffer, pageID, PageType.Data) { } public DataBlock GetBlock(byte index) { BufferSlice segment = Get(index); return new DataBlock(this, index, segment); } public DataBlock InsertBlock(int bytesLength, bool extend) { byte index; BufferSlice segment = Insert((ushort)(bytesLength + 6), out index); return new DataBlock(this, index, segment, extend, PageAddress.Empty); } public DataBlock UpdateBlock(DataBlock currentBlock, int bytesLength) { BufferSlice segment = Update(currentBlock.Position.Index, (ushort)(bytesLength + 6)); return new DataBlock(this, currentBlock.Position.Index, segment, currentBlock.Extend, currentBlock.NextBlock); } public void DeleteBlock(byte index) { Delete(index); } public IEnumerable GetBlocks() { foreach (byte usedIndex in GetUsedIndexs()) { int offset = BasePage.CalcPositionAddr(usedIndex); ushort offset2 = _buffer.ReadUInt16(offset); if (!_buffer.ReadBool(offset2)) { yield return new PageAddress(base.PageID, usedIndex); } } } public static byte FreeIndexSlot(int freeBytes) { Constants.ENSURE(freeBytes >= 0, "FreeBytes must be positive: {0}", freeBytes); for (int i = 0; i < _freePageSlots.Length; i++) { if (freeBytes >= _freePageSlots[i]) { return (byte)i; } } return 4; } public static int GetMinimumIndexSlot(int length) { return FreeIndexSlot(length) - 1; } } internal class HeaderPage : BasePage { public const string HEADER_INFO = "** This is a LiteDB file **"; public const byte FILE_VERSION = 8; public const int P_HEADER_INFO = 32; public const int P_FILE_VERSION = 59; public const int P_FREE_EMPTY_PAGE_ID = 60; public const int P_LAST_PAGE_ID = 64; public const int P_CREATION_TIME = 68; public const int P_INVALID_DATAFILE_STATE = 191; public const int P_COLLECTIONS = 192; public const int COLLECTIONS_SIZE = 8000; private BsonDocument _collections; private bool _isCollectionsChanged; public uint FreeEmptyPageList { get; set; } public uint LastPageID { get; set; } public DateTime CreationTime { get; } public EnginePragmas Pragmas { get; set; } public HeaderPage(PageBuffer buffer, uint pageID) : base(buffer, 0u, PageType.Header) { CreationTime = DateTime.UtcNow; FreeEmptyPageList = uint.MaxValue; LastPageID = 0u; Pragmas = new EnginePragmas(this); _buffer.Write("** This is a LiteDB file **", 32); ((BufferSlice)_buffer).Write((byte)8, 59); _buffer.Write(CreationTime, 68); _collections = new BsonDocument(); } public HeaderPage(PageBuffer buffer) : base(buffer) { CreationTime = _buffer.ReadDateTime(68); LoadPage(); } private void LoadPage() { string strA = _buffer.ReadString(32, "** This is a LiteDB file **".Length); byte b = _buffer[59]; if (string.CompareOrdinal(strA, "** This is a LiteDB file **") != 0 || b != 8) { throw LiteException.InvalidDatabase(); } FreeEmptyPageList = _buffer.ReadUInt32(60); LastPageID = _buffer.ReadUInt32(64); Pragmas = new EnginePragmas(_buffer, this); BufferSlice bufferSlice = _buffer.Slice(192, 8000); using (BufferReader bufferReader = new BufferReader(new BufferSlice[1] { bufferSlice })) { _collections = bufferReader.ReadDocument().GetValue(); } _isCollectionsChanged = false; } public override PageBuffer UpdateBuffer() { _buffer.Write(FreeEmptyPageList, 60); _buffer.Write(LastPageID, 64); Pragmas.UpdateBuffer(_buffer); if (_isCollectionsChanged) { using (BufferWriter bufferWriter = new BufferWriter(_buffer.Slice(192, 8000))) { bufferWriter.WriteDocument(_collections, recalc: true); } _isCollectionsChanged = false; } return base.UpdateBuffer(); } public PageBuffer Savepoint() { UpdateBuffer(); PageBuffer pageBuffer = new PageBuffer(new byte[8192], 0, 0); System.Buffer.BlockCopy(_buffer.Array, _buffer.Offset, pageBuffer.Array, pageBuffer.Offset, 8192); return pageBuffer; } public void Restore(PageBuffer savepoint) { System.Buffer.BlockCopy(savepoint.Array, savepoint.Offset, _buffer.Array, _buffer.Offset, 8192); LoadPage(); } public uint GetCollectionPageID(string collection) { if (_collections.TryGetValue(collection, out var value)) { return (uint)value.AsInt32; } return uint.MaxValue; } public IEnumerable> GetCollections() { foreach (KeyValuePair element in _collections.GetElements()) { yield return new KeyValuePair(element.Key, (uint)element.Value.AsInt32); } } public void InsertCollection(string name, uint pageID) { _collections[name] = (int)pageID; _isCollectionsChanged = true; } public void DeleteCollection(string name) { _collections.Remove(name); _isCollectionsChanged = true; } public void RenameCollection(string oldName, string newName) { BsonValue value = _collections[oldName]; _collections.Remove(oldName); _collections.Add(newName, value); _isCollectionsChanged = true; } public int GetAvailableCollectionSpace() { return 8000 - _collections.GetBytesCount(recalc: true) - 1 - 1 - 4 - 8; } } internal class IndexPage : BasePage { public IndexPage(PageBuffer buffer) : base(buffer) { Constants.ENSURE(base.PageType == PageType.Index, "page type must be index page"); if (base.PageType != PageType.Index) { throw LiteException.InvalidPageType(PageType.Index, this); } } public IndexPage(PageBuffer buffer, uint pageID) : base(buffer, pageID, PageType.Index) { } public IndexNode GetIndexNode(byte index) { BufferSlice segment = Get(index); return new IndexNode(this, index, segment); } public IndexNode InsertIndexNode(byte slot, byte level, BsonValue key, PageAddress dataBlock, int bytesLength) { byte index; BufferSlice segment = Insert((ushort)bytesLength, out index); return new IndexNode(this, index, segment, slot, level, key, dataBlock); } public void DeleteIndexNode(byte index) { Delete(index); } public IEnumerable GetIndexNodes() { foreach (byte usedIndex in GetUsedIndexs()) { yield return GetIndexNode(usedIndex); } } public static byte FreeIndexSlot(int freeBytes) { Constants.ENSURE(freeBytes >= 0, "freeBytes must be positive"); return (freeBytes < 1400) ? ((byte)1) : ((byte)0); } } public static class Pragmas { public const string USER_VERSION = "USER_VERSION"; public const string COLLATION = "COLLATION"; public const string TIMEOUT = "TIMEOUT"; public const string LIMIT_SIZE = "LIMIT_SIZE"; public const string UTC_DATE = "UTC_DATE"; public const string CHECKPOINT = "CHECKPOINT"; } internal abstract class Index { public string Name { get; private set; } public int Order { get; set; } internal Index(string name, int order) { Name = name; Order = order; } public abstract uint GetCost(CollectionIndex index); public abstract IEnumerable Execute(IndexService indexer, CollectionIndex index); public virtual IEnumerable Run(CollectionPage col, IndexService indexer) { CollectionIndex collectionIndex = col.GetCollectionIndex(Name); if (collectionIndex == null) { throw LiteException.IndexNotFound(Name); } return Execute(indexer, collectionIndex).DistinctBy((IndexNode x) => x.DataBlock, null); } } internal class IndexAll : Index { public IndexAll(string name, int order) : base(name, order) { } public override uint GetCost(CollectionIndex index) { return 100u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { return indexer.FindAll(index, base.Order); } public override string ToString() { return $"FULL INDEX SCAN({base.Name})"; } } internal class IndexEquals : Index { private readonly BsonValue _value; public IndexEquals(string name, BsonValue value) : base(name, 1) { _value = value; } public override uint GetCost(CollectionIndex index) { if (index.Unique) { return 1u; } return 10u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { IndexNode node2 = indexer.Find(index, _value, sibling: false, 1); if (node2 == null) { yield break; } yield return node2; if (index.Unique) { yield break; } IndexNode first = node2; while (!node2.Next[0].IsEmpty) { IndexNode node3; node2 = (node3 = indexer.GetNode(node2.Next[0])); if (node3.Key.CompareTo(_value, indexer.Collation) != 0 || node2.Key.IsMinValue || node2.Key.IsMaxValue) { break; } yield return node2; } node2 = first; while (!node2.Prev[0].IsEmpty) { IndexNode node3; node2 = (node3 = indexer.GetNode(node2.Prev[0])); if (node3.Key.CompareTo(_value, indexer.Collation) == 0 && !node2.Key.IsMinValue && !node2.Key.IsMaxValue) { yield return node2; continue; } break; } } public override string ToString() { return $"INDEX SEEK({base.Name} = {_value})"; } } internal class IndexIn : Index { private readonly BsonArray _values; public IndexIn(string name, BsonArray values, int order) : base(name, order) { _values = values; } public override uint GetCost(CollectionIndex index) { if (!index.Unique) { return (uint)(_values.Count * 10); } return (uint)_values.Count; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { foreach (BsonValue item in _values.Distinct()) { IndexEquals indexEquals = new IndexEquals(base.Name, item); foreach (IndexNode item2 in indexEquals.Execute(indexer, index)) { yield return item2; } } } public override string ToString() { return $"INDEX SEEK({base.Name} IN {JsonSerializer.Serialize(_values)})"; } } internal class IndexLike : Index { private readonly string _startsWith; private readonly bool _equals; private readonly bool _testSqlLike; private readonly string _pattern; public IndexLike(string name, BsonValue value, int order) : base(name, order) { _pattern = value.AsString; _startsWith = _pattern.SqlLikeStartsWith(out _testSqlLike); _equals = _pattern == _startsWith; } public override uint GetCost(CollectionIndex index) { if (_startsWith.Length > 0) { return 10u; } return 100u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { if (_startsWith.Length <= 0) { return ExecuteLike(indexer, index); } return ExecuteStartsWith(indexer, index); } private IEnumerable ExecuteStartsWith(IndexService indexer, CollectionIndex index) { IndexNode first = indexer.Find(index, _startsWith, sibling: true, base.Order); IndexNode node = first; if (first == null) { yield break; } while (node != null && !node.Key.IsMinValue && !node.Key.IsMaxValue) { string text = (node.Key.IsString ? node.Key.AsString : (node.Key.IsNull ? "" : node.Key.ToString())); if (!(_equals ? text.Equals(_startsWith, StringComparison.OrdinalIgnoreCase) : text.StartsWith(_startsWith, StringComparison.OrdinalIgnoreCase))) { break; } if (!_testSqlLike || (_testSqlLike && text.SqlLike(_pattern, indexer.Collation))) { yield return node; } node = indexer.GetNode(node.GetNextPrev(0, -base.Order)); } node = indexer.GetNode(first.GetNextPrev(0, base.Order)); while (node != null && !node.Key.IsMinValue && !node.Key.IsMaxValue) { string text2 = (node.Key.IsString ? node.Key.AsString : (node.Key.IsNull ? "" : node.Key.ToString())); if (_equals ? text2.Equals(_pattern, StringComparison.OrdinalIgnoreCase) : text2.StartsWith(_startsWith, StringComparison.OrdinalIgnoreCase)) { if (!node.DataBlock.IsEmpty && (!_testSqlLike || (_testSqlLike && text2.SqlLike(_pattern, indexer.Collation)))) { yield return node; } node = indexer.GetNode(node.GetNextPrev(0, base.Order)); continue; } break; } } private IEnumerable ExecuteLike(IndexService indexer, CollectionIndex index) { return from x in indexer.FindAll(index, base.Order) where x.Key.IsString && x.Key.AsString.SqlLike(_pattern, indexer.Collation) select x; } public override string ToString() { return string.Format("{0}({1} LIKE \"{2}\")", (_startsWith.Length > 0) ? "INDEX SEEK (+RANGE SCAN)" : "FULL INDEX SCAN", base.Name, _pattern); } } internal class IndexRange : Index { private readonly BsonValue _start; private readonly BsonValue _end; private readonly bool _startEquals; private readonly bool _endEquals; public IndexRange(string name, BsonValue start, BsonValue end, bool startEquals, bool endEquals, int order) : base(name, order) { _start = start; _end = end; _startEquals = startEquals; _endEquals = endEquals; } public override uint GetCost(CollectionIndex index) { return 20u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { BsonValue start = ((base.Order == 1) ? _start : _end); BsonValue end = ((base.Order == 1) ? _end : _start); bool startEquals = ((base.Order == 1) ? _startEquals : _endEquals); bool endEquals = ((base.Order == 1) ? _endEquals : _startEquals); IndexNode first = ((start.Type == BsonType.MinValue) ? indexer.GetNode(index.Head) : ((start.Type == BsonType.MaxValue) ? indexer.GetNode(index.Tail) : indexer.Find(index, start, sibling: true, base.Order))); IndexNode node = first; if (startEquals && node != null) { while (!node.GetNextPrev(0, -base.Order).IsEmpty) { IndexNode node2; node = (node2 = indexer.GetNode(node.GetNextPrev(0, -base.Order))); if (node2.Key.CompareTo(start) != 0 || node.Key.IsMinValue || node.Key.IsMaxValue) { break; } yield return node; } node = first; } while (node != null && node.Key.CompareTo(start, indexer.Collation) == 0) { if (startEquals && !node.Key.IsMinValue && !node.Key.IsMaxValue) { yield return node; } node = indexer.GetNode(node.GetNextPrev(0, base.Order)); } while (node != null) { int num = node.Key.CompareTo(end, indexer.Collation); if (endEquals && num == 0 && !node.Key.IsMinValue && !node.Key.IsMaxValue) { yield return node; } else { if (num != -base.Order || node.Key.IsMinValue || node.Key.IsMaxValue) { break; } yield return node; } node = indexer.GetNode(node.GetNextPrev(0, base.Order)); } } public override string ToString() { if (_start.IsMinValue && !_endEquals) { return $"INDEX SCAN({base.Name} < {_end})"; } if (_start.IsMinValue && _endEquals) { return $"INDEX SCAN({base.Name} <= {_end})"; } if (_end.IsMaxValue && !_startEquals) { return $"INDEX SCAN({base.Name} > {_start})"; } if (_end.IsMaxValue && _startEquals) { return $"INDEX SCAN({base.Name} >= {_start})"; } return $"INDEX RANGE SCAN({base.Name} BETWEEN {_start} AND {_end})"; } } internal class IndexScan : Index { private readonly Func _func; public IndexScan(string name, Func func, int order) : base(name, order) { _func = func; } public override uint GetCost(CollectionIndex index) { return 80u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { return from i in indexer.FindAll(index, base.Order) where _func(i.Key) select i; } public override string ToString() { return $"FULL INDEX SCAN({base.Name})"; } } internal class IndexVirtual : Index, IDocumentLookup { private readonly IEnumerable _source; private Dictionary _cache = new Dictionary(); public IndexVirtual(IEnumerable source) : base(null, 0) { _source = source; } public override uint GetCost(CollectionIndex index) { return 100u; } public override IEnumerable Execute(IndexService indexer, CollectionIndex index) { throw new NotImplementedException(); } public override IEnumerable Run(CollectionPage col, IndexService indexer) { uint rawId = 0u; foreach (BsonDocument item in _source) { rawId++; item.RawId = new PageAddress(rawId, 0); if (_cache != null) { _cache[rawId] = item; if (_cache.Count > 2000) { _cache = null; } } yield return new IndexNode(item); } } public BsonDocument Load(IndexNode node) { return node.Key as BsonDocument; } public BsonDocument Load(PageAddress rawId) { if (_cache == null) { throw new LiteException(0, $"OrderBy/GroupBy operation are supported only in virtual collection with less than {2000} documents"); } return _cache[rawId.PageID]; } public override string ToString() { return $"FULL COLLECTION SCAN"; } } internal class DatafileLookup : IDocumentLookup { protected readonly DataService _data; protected readonly bool _utcDate; protected readonly HashSet _fields; public DatafileLookup(DataService data, bool utcDate, HashSet fields) { _data = data; _utcDate = utcDate; _fields = fields; } public virtual BsonDocument Load(IndexNode node) { Constants.ENSURE(node.DataBlock != PageAddress.Empty, "data block must be a valid block address"); return Load(node.DataBlock); } public virtual BsonDocument Load(PageAddress rawId) { using BufferReader bufferReader = new BufferReader(_data.Read(rawId), _utcDate); BsonDocument value = bufferReader.ReadDocument(_fields).GetValue(); value.RawId = rawId; return value; } } internal interface IDocumentLookup { BsonDocument Load(IndexNode node); BsonDocument Load(PageAddress rawId); } internal class IndexLookup : IDocumentLookup { private readonly IndexService _indexer; private readonly string _name; public IndexLookup(IndexService indexer, string name) { _indexer = indexer; _name = name; } public BsonDocument Load(IndexNode node) { Constants.ENSURE(!node.DataBlock.IsEmpty, "Never should be empty rawid"); return new BsonDocument { [_name] = node.Key, RawId = node.DataBlock }; } public BsonDocument Load(PageAddress rawId) { IndexNode node = _indexer.GetNode(rawId); return Load(node); } } internal abstract class BasePipe { protected readonly TransactionService _transaction; protected readonly IDocumentLookup _lookup; protected readonly SortDisk _tempDisk; protected readonly EnginePragmas _pragmas; protected readonly uint _maxItemsCount; public BasePipe(TransactionService transaction, IDocumentLookup lookup, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount) { _transaction = transaction; _lookup = lookup; _tempDisk = tempDisk; _pragmas = pragmas; _maxItemsCount = maxItemsCount; } public abstract IEnumerable Pipe(IEnumerable nodes, QueryPlan query); protected IEnumerable LoadDocument(IEnumerable nodes) { foreach (IndexNode node in nodes) { yield return _lookup.Load(node); _transaction.Safepoint(); } } protected IEnumerable Include(IEnumerable source, BsonExpression path) { string last = null; Snapshot snapshot = null; IndexService indexer = null; DataService data = null; CollectionIndex index = null; IDocumentLookup lookup = null; foreach (BsonDocument item in source) { foreach (BsonValue item2 in (from x in path.Execute(item, _pragmas.Collation) where x.IsDocument || x.IsArray select x).ToList()) { if (item2.IsDocument) { DoInclude(item2.AsDocument); continue; } foreach (BsonDocument item3 in from x in item2.AsArray where x.IsDocument select x.AsDocument) { DoInclude(item3); } } yield return item; } void DoInclude(BsonDocument value) { BsonValue bsonValue = value["$id"]; BsonValue bsonValue2 = value["$ref"]; if (!bsonValue.IsNull && bsonValue2.IsString) { if (last != bsonValue2.AsString) { last = bsonValue2.AsString; snapshot = _transaction.CreateSnapshot(LockMode.Read, last, addIfNotExists: false); indexer = new IndexService(snapshot, _pragmas.Collation, _maxItemsCount); data = new DataService(snapshot, _maxItemsCount); lookup = new DatafileLookup(data, _pragmas.UtcDate, null); index = snapshot.CollectionPage?.PK; } if (index != null) { IndexNode indexNode = indexer.Find(index, bsonValue, sibling: false, 1); if (indexNode != null) { BsonDocument source2 = lookup.Load(indexNode); value.Remove("$ref"); { foreach (KeyValuePair item4 in source2.Where((KeyValuePair x) => x.Key != "_id")) { value[item4.Key] = item4.Value; } return; } } value["$missing"] = true; } } } } protected IEnumerable Filter(IEnumerable source, BsonExpression expr) { foreach (BsonDocument item in source) { BsonValue bsonValue = expr.ExecuteScalar(item, _pragmas.Collation); if (bsonValue.IsBoolean && bsonValue.AsBoolean) { yield return item; } } } protected IEnumerable OrderBy(IEnumerable source, BsonExpression expr, int order, int offset, int limit) { IEnumerable> items = source.Select((BsonDocument x) => new KeyValuePair(expr.ExecuteScalar(x, _pragmas.Collation), x.RawId)); using SortService sorter = new SortService(_tempDisk, order, _pragmas); sorter.Insert(items); IEnumerable> enumerable = sorter.Sort().Skip(offset).Take(limit); foreach (KeyValuePair item in enumerable) { yield return _lookup.Load(item.Value); } } } internal class DocumentCacheEnumerable : IEnumerable, IEnumerable, IDisposable { private IEnumerator _enumerator; private readonly List _cache = new List(); private readonly IDocumentLookup _lookup; public DocumentCacheEnumerable(IEnumerable source, IDocumentLookup lookup) { _enumerator = source.GetEnumerator(); _lookup = lookup; } public void Dispose() { if (_enumerator != null) { while (_enumerator.MoveNext()) { } _enumerator.Dispose(); _enumerator = null; } } public IEnumerator GetEnumerator() { int index; for (index = 0; index < _cache.Count; index++) { PageAddress rawId = _cache[index]; yield return _lookup.Load(rawId); } while (_enumerator != null && _enumerator.MoveNext()) { BsonDocument current = _enumerator.Current; Constants.ENSURE(!current.RawId.IsEmpty, "rawId must have a valid value"); _cache.Add(current.RawId); yield return current; index++; } if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; } for (; index < _cache.Count; index++) { PageAddress rawId2 = _cache[index]; yield return _lookup.Load(rawId2); } } IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } } internal class GroupByPipe : BasePipe { public GroupByPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount) : base(transaction, loader, tempDisk, pragmas, maxItemsCount) { } public override IEnumerable Pipe(IEnumerable nodes, QueryPlan query) { IEnumerable source = LoadDocument(nodes); foreach (BsonExpression filter in query.Filters) { source = Filter(source, filter); } if (query.OrderBy != null) { source = OrderBy(source, query.OrderBy.Expression, query.OrderBy.Order, 0, int.MaxValue); } IEnumerable groups = GroupBy(source, query.GroupBy); IEnumerable enumerable = SelectGroupBy(groups, query.GroupBy); if (query.Offset > 0) { enumerable = enumerable.Skip(query.Offset); } if (query.Limit < int.MaxValue) { enumerable = enumerable.Take(query.Limit); } return enumerable; } private IEnumerable GroupBy(IEnumerable source, GroupBy groupBy) { using IEnumerator enumerator = source.GetEnumerator(); Done done = new Done { Running = enumerator.MoveNext() }; while (done.Running) { BsonValue bsonValue = groupBy.Expression.ExecuteScalar(enumerator.Current, _pragmas.Collation); groupBy.Select.Parameters["key"] = bsonValue; IEnumerable source2 = YieldDocuments(bsonValue, enumerator, groupBy, done); yield return new DocumentCacheEnumerable(source2, _lookup); } } private IEnumerable YieldDocuments(BsonValue key, IEnumerator enumerator, GroupBy groupBy, Done done) { yield return enumerator.Current; while (done.Running = enumerator.MoveNext()) { BsonValue bsonValue = groupBy.Expression.ExecuteScalar(enumerator.Current, _pragmas.Collation); if (key == bsonValue) { yield return enumerator.Current; continue; } groupBy.Select.Parameters["key"] = bsonValue; break; } } private IEnumerable SelectGroupBy(IEnumerable groups, GroupBy groupBy) { string defaultName = groupBy.Select.DefaultFieldName(); foreach (DocumentCacheEnumerable group in groups) { BsonValue bsonValue2; try { if (groupBy.Having == null) { goto IL_00b2; } BsonValue bsonValue = groupBy.Having.ExecuteScalar(group, null, null, _pragmas.Collation); if (bsonValue.IsBoolean && bsonValue.AsBoolean) { goto IL_00b2; } goto end_IL_006e; IL_00b2: bsonValue2 = groupBy.Select.ExecuteScalar(group, null, null, _pragmas.Collation); goto IL_00db; end_IL_006e:; } finally { group.Dispose(); } continue; IL_00db: if (bsonValue2.IsDocument) { yield return bsonValue2.AsDocument; continue; } yield return new BsonDocument { [defaultName] = bsonValue2 }; } } } internal class QueryPipe : BasePipe { public QueryPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount) : base(transaction, loader, tempDisk, pragmas, maxItemsCount) { } public override IEnumerable Pipe(IEnumerable nodes, QueryPlan query) { IEnumerable source = LoadDocument(nodes); foreach (BsonExpression item in query.IncludeBefore) { source = Include(source, item); } foreach (BsonExpression filter in query.Filters) { source = Filter(source, filter); } if (query.OrderBy != null) { source = OrderBy(source, query.OrderBy.Expression, query.OrderBy.Order, query.Offset, query.Limit); } else { if (query.Offset > 0) { source = source.Skip(query.Offset); } if (query.Limit < int.MaxValue) { source = source.Take(query.Limit); } } foreach (BsonExpression item2 in query.IncludeAfter) { source = Include(source, item2); } if (query.Select.All) { return SelectAll(source, query.Select.Expression); } return Select(source, query.Select.Expression); } private IEnumerable Select(IEnumerable source, BsonExpression select) { string defaultName = select.DefaultFieldName(); foreach (BsonDocument item in source) { BsonValue bsonValue = select.ExecuteScalar(item, _pragmas.Collation); if (bsonValue.IsDocument) { yield return bsonValue.AsDocument; continue; } yield return new BsonDocument { [defaultName] = bsonValue }; } } private IEnumerable SelectAll(IEnumerable source, BsonExpression select) { DocumentCacheEnumerable source2 = new DocumentCacheEnumerable(source, _lookup); string defaultName = select.DefaultFieldName(); IEnumerable enumerable = select.Execute(source2, _pragmas.Collation); foreach (BsonValue item in enumerable) { if (item.IsDocument) { yield return item.AsDocument; continue; } yield return new BsonDocument { [defaultName] = item }; } } } internal class QueryExecutor { private readonly LiteEngine _engine; private readonly EngineState _state; private readonly TransactionMonitor _monitor; private readonly SortDisk _sortDisk; private readonly DiskService _disk; private readonly EnginePragmas _pragmas; private readonly CursorInfo _cursor; private readonly string _collection; private readonly Query _query; private readonly IEnumerable _source; public QueryExecutor(LiteEngine engine, EngineState state, TransactionMonitor monitor, SortDisk sortDisk, DiskService disk, EnginePragmas pragmas, string collection, Query query, IEnumerable source) { _engine = engine; _state = state; _monitor = monitor; _sortDisk = sortDisk; _disk = disk; _pragmas = pragmas; _collection = collection; _query = query; _cursor = new CursorInfo(collection, query); _source = source; } public BsonDataReader ExecuteQuery() { if (_query.Into == null) { return ExecuteQuery(_query.ExplainPlan); } return ExecuteQueryInto(_query.Into, _query.IntoAutoId); } internal BsonDataReader ExecuteQuery(bool executionPlan) { bool isNew; TransactionService transaction = _monitor.GetTransaction(create: true, queryOnly: true, out isNew); transaction.OpenCursors.Add(_cursor); IEnumerable source = RunQuery(); source = source.OnDispose(delegate { transaction.OpenCursors.Remove(_cursor); }); if (isNew) { source = source.OnDispose(delegate { _monitor.ReleaseTransaction(transaction); }); } return new BsonDataReader(source, _collection, _state); IEnumerable RunQuery() { Snapshot snapshot = transaction.CreateSnapshot(_query.ForUpdate ? LockMode.Write : LockMode.Read, _collection, addIfNotExists: false); if (snapshot.CollectionPage == null && _source == null) { if (_query.Select.UseSource) { yield return _query.Select.ExecuteScalar(_pragmas.Collation).AsDocument; } } else { QueryPlan queryPlan = new QueryOptimization(snapshot, _query, _source, _pragmas.Collation).ProcessQuery(); queryPlan.GetExecutionPlan(); if (!executionPlan) { IEnumerable nodes = queryPlan.Index.Run(snapshot.CollectionPage, new IndexService(snapshot, _pragmas.Collation, _disk.MAX_ITEMS_COUNT)); BasePipe pipe = queryPlan.GetPipe(transaction, snapshot, _sortDisk, _pragmas, _disk.MAX_ITEMS_COUNT); using (_cursor.Elapsed.StartDisposable()) { using (IEnumerator enumerator = pipe.Pipe(nodes, queryPlan).GetEnumerator()) { bool flag; try { flag = enumerator.MoveNext(); } catch (Exception ex) { _state.Handle(ex); throw ex; } while (flag) { _cursor.Fetched++; _cursor.Elapsed.Stop(); yield return enumerator.Current; if (transaction.State != 0) { throw new LiteException(0, "There is no more active transaction for this cursor: " + _cursor.Query.ToSQL(_cursor.Collection)); } _cursor.Elapsed.Start(); try { flag = enumerator.MoveNext(); } catch (Exception ex2) { _state.Handle(ex2); throw ex2; } } } yield break; } } yield return queryPlan.GetExecutionPlan(); } } } internal BsonDataReader ExecuteQueryInto(string into, BsonAutoId autoId) { int num; if (into.StartsWith("$")) { SqlParser.ParseCollection(new Tokenizer(into), out var name, out var options); num = _engine.GetSystemCollection(name).Output(GetResultset(), options); } else { num = _engine.Insert(into, GetResultset(), autoId); } return new BsonDataReader(num); IEnumerable GetResultset() { using BsonDataReader reader = ExecuteQuery(executionPlan: false); while (reader.Read()) { yield return reader.Current.AsDocument; } } } } internal class QueryOptimization { private readonly Snapshot _snapshot; private readonly Query _query; private readonly Collation _collation; private readonly QueryPlan _queryPlan; private readonly List _terms = new List(); public QueryOptimization(Snapshot snapshot, Query query, IEnumerable source, Collation collation) { if (query.Select == null) { throw new ArgumentNullException("Select"); } _snapshot = snapshot; _query = query; _collation = collation; _queryPlan = new QueryPlan(snapshot.CollectionName) { Index = ((source != null) ? new IndexVirtual(source) : null), Select = new Select(_query.Select, _query.Select.UseSource), ForUpdate = query.ForUpdate, Limit = query.Limit, Offset = query.Offset }; } public QueryPlan ProcessQuery() { SplitWherePredicateInTerms(); OptimizeTerms(); DefineQueryFields(); DefineIndex(); DefineOrderBy(); DefineGroupBy(); DefineIncludes(); return _queryPlan; } private void SplitWherePredicateInTerms() { foreach (BsonExpression item in _query.Where) { add(item); } void add(BsonExpression predicate) { if (predicate.UseSource) { throw new LiteException(0, "WHERE filter can not use `*` expression in `" + predicate.Source); } if (predicate.IsPredicate || predicate.Type == BsonExpressionType.Or) { _terms.Add(predicate); } else { if (predicate.Type != BsonExpressionType.And) { throw LiteException.InvalidExpressionTypePredicate(predicate); } BsonExpression left = predicate.Left; BsonExpression right = predicate.Right; add(left); add(right); } } } private void OptimizeTerms() { for (int i = 0; i < _terms.Count; i++) { BsonExpression bsonExpression = _terms[i]; BsonExpression left = bsonExpression.Left; if (left != null && !left.IsScalar && bsonExpression.IsANY && bsonExpression.Type == BsonExpressionType.Equal) { BsonExpression right = bsonExpression.Right; if (right != null && right.Type == BsonExpressionType.Path) { _terms[i] = BsonExpression.Create(bsonExpression.Right.Source + " IN ARRAY(" + bsonExpression.Left.Source + ")", bsonExpression.Parameters); } } } } private void DefineQueryFields() { HashSet hashSet = new HashSet(StringComparer.OrdinalIgnoreCase); hashSet.AddRange(_query.Select.Fields); hashSet.AddRange(_terms.SelectMany((BsonExpression x) => x.Fields)); hashSet.AddRange(_query.Includes.SelectMany((BsonExpression x) => x.Fields)); hashSet.AddRange(_query.GroupBy?.Fields); hashSet.AddRange(_query.Having?.Fields); hashSet.AddRange(_query.OrderBy?.Fields); if (hashSet.Contains("$")) { hashSet.Clear(); } _queryPlan.Fields = hashSet; } private void DefineIndex() { BsonExpression selected = null; if (_queryPlan.Index == null) { IndexCost indexCost = ChooseIndex(_queryPlan.Fields); if (indexCost != null) { _queryPlan.Index = indexCost.Index; _queryPlan.IndexCost = indexCost.Cost; _queryPlan.IndexExpression = indexCost.IndexExpression; } else { CollectionIndex pK = _snapshot.CollectionPage.PK; _queryPlan.Index = new IndexAll("_id", 1); _queryPlan.IndexCost = _queryPlan.Index.GetCost(pK); _queryPlan.IndexExpression = "$._id"; } selected = indexCost?.Expression; } else { Constants.ENSURE(_queryPlan.Index is IndexVirtual, "pre-defined index must be only for virtual collections"); _queryPlan.IndexCost = 0u; } if (_queryPlan.Fields.Count == 1 && _queryPlan.IndexExpression == "$." + _queryPlan.Fields.First()) { _queryPlan.IsIndexKeyOnly = true; } _queryPlan.Filters.AddRange(_terms.Where((BsonExpression x) => x != selected)); } private IndexCost ChooseIndex(HashSet fields) { CollectionIndex[] source = _snapshot.CollectionPage.GetCollectionIndexes().ToArray(); string preferred = ((fields.Count == 1) ? ("$." + fields.First()) : null); IndexCost indexCost = null; foreach (BsonExpression expr in _terms.Where((BsonExpression x) => x.IsPredicate)) { Constants.ENSURE(expr.Left != null && expr.Right != null, "predicate expression must has left/right expressions"); Tuple tuple = null; if (!expr.Left.IsScalar && expr.Right.IsScalar) { if (expr.IsANY) { tuple = (from x in source where x.Expression == expr.Left.Source && expr.Right.IsValue select Tuple.Create(x, expr.Right)).FirstOrDefault(); } } else { tuple = (from x in source where x.Expression == expr.Left.Source && expr.Right.IsValue select Tuple.Create(x, expr.Right)).Union(from x in source where x.Expression == expr.Right.Source && expr.Left.IsValue select Tuple.Create(x, expr.Left)).FirstOrDefault(); } if (tuple != null) { IndexCost indexCost2 = new IndexCost(tuple.Item1, expr, tuple.Item2, _collation); if (indexCost == null || indexCost2.Cost < indexCost.Cost) { indexCost = indexCost2; } } } if (indexCost == null && (_query.OrderBy != null || _query.GroupBy != null || preferred != null)) { CollectionIndex collectionIndex = source.FirstOrDefault((CollectionIndex x) => x.Expression == _query.GroupBy?.Source) ?? source.FirstOrDefault((CollectionIndex x) => x.Expression == _query.OrderBy?.Source) ?? source.FirstOrDefault((CollectionIndex x) => x.Expression == preferred); if (collectionIndex != null) { indexCost = new IndexCost(collectionIndex); } } return indexCost; } private void DefineOrderBy() { if (_query.OrderBy != null) { OrderBy orderBy = new OrderBy(_query.OrderBy, _query.Order); if (orderBy.Expression.Source == _queryPlan.IndexExpression) { _queryPlan.Index.Order = orderBy.Order; orderBy = null; } _queryPlan.OrderBy = orderBy; } } private void DefineGroupBy() { if (_query.GroupBy != null) { if (_query.OrderBy != null) { throw new NotSupportedException("GROUP BY expression do not support ORDER BY"); } if (_query.Includes.Count > 0) { throw new NotSupportedException("GROUP BY expression do not support INCLUDE"); } GroupBy groupBy = new GroupBy(_query.GroupBy, _queryPlan.Select.Expression, _query.Having); OrderBy orderBy = null; if (!(groupBy.Expression.Source == _queryPlan.IndexExpression)) { orderBy = new OrderBy(groupBy.Expression, 1); } _queryPlan.GroupBy = groupBy; _queryPlan.OrderBy = orderBy; } } private void DefineIncludes() { foreach (BsonExpression include in _query.Includes) { string field = include.Fields.Single(); int num; if (!_queryPlan.Filters.Any((BsonExpression x) => x.Fields.Contains(field))) { OrderBy orderBy = _queryPlan.OrderBy; if (orderBy == null) { num = 0; } else { num = (orderBy.Expression.Fields.Contains(field) ? 1 : 0); if (num != 0) { goto IL_0080; } } goto IL_0091; } num = 1; goto IL_0080; IL_0080: _queryPlan.IncludeBefore.Add(include); goto IL_0091; IL_0091: if (num == 0 || _queryPlan.OrderBy != null) { _queryPlan.IncludeAfter.Add(include); } } } } internal class GroupBy { public BsonExpression Expression { get; } public BsonExpression Select { get; } public BsonExpression Having { get; } public GroupBy(BsonExpression expression, BsonExpression select, BsonExpression having) { Expression = expression; Select = select; Having = having; } } internal class IndexCost { public uint Cost { get; } public BsonExpression Expression { get; } public string IndexExpression { get; } public Index Index { get; } public IndexCost(CollectionIndex index, BsonExpression expr, BsonExpression value, Collation collation) { IndexCost indexCost = this; IndexExpression = index.Expression; Expression = expr; BsonExpressionType exprType = expr.Type; if (expr.Left.IsValue) { switch (expr.Type) { case BsonExpressionType.GreaterThan: exprType = BsonExpressionType.LessThan; break; case BsonExpressionType.GreaterThanOrEqual: exprType = BsonExpressionType.LessThanOrEqual; break; case BsonExpressionType.LessThan: exprType = BsonExpressionType.GreaterThan; break; case BsonExpressionType.LessThanOrEqual: exprType = BsonExpressionType.GreaterThanOrEqual; break; } } Index = (from x in value.Execute(collation) select indexCost.CreateIndex(exprType, index.Name, x)).FirstOrDefault(); Constants.ENSURE(Index != null, "index must be not null"); Cost = Index.GetCost(index); } public IndexCost(CollectionIndex index) { Expression = BsonExpression.Create(index.Expression); Index = new IndexAll(index.Name, 1); Cost = Index.GetCost(index); IndexExpression = index.Expression; } private Index CreateIndex(BsonExpressionType type, string name, BsonValue value) { switch (type) { case BsonExpressionType.Equal: return new IndexEquals(name, value); case BsonExpressionType.Between: return new IndexRange(name, value.AsArray[0], value.AsArray[1], startEquals: true, endEquals: true, 1); case BsonExpressionType.Like: return new IndexLike(name, value.AsString, 1); case BsonExpressionType.GreaterThan: return new IndexRange(name, value, BsonValue.MaxValue, startEquals: false, endEquals: true, 1); case BsonExpressionType.GreaterThanOrEqual: return new IndexRange(name, value, BsonValue.MaxValue, startEquals: true, endEquals: true, 1); case BsonExpressionType.LessThan: return new IndexRange(name, BsonValue.MinValue, value, startEquals: true, endEquals: false, 1); case BsonExpressionType.LessThanOrEqual: return new IndexRange(name, BsonValue.MinValue, value, startEquals: true, endEquals: true, 1); case BsonExpressionType.NotEqual: return new IndexScan(name, (BsonValue x) => x.CompareTo(value) != 0, 1); case BsonExpressionType.In: if (!value.IsArray) { return new IndexEquals(name, value); } return new IndexIn(name, value.AsArray, 1); default: return null; } } } internal class OrderBy { public BsonExpression Expression { get; } public int Order { get; set; } public OrderBy(BsonExpression expression, int order) { Expression = expression; Order = order; } } internal class QueryPlan { public string Collection { get; set; } public Index Index { get; set; } public string IndexExpression { get; set; } public uint IndexCost { get; internal set; } public bool IsIndexKeyOnly { get; set; } public List Filters { get; set; } = new List(); public List IncludeBefore { get; set; } = new List(); public List IncludeAfter { get; set; } = new List(); public OrderBy OrderBy { get; set; } public GroupBy GroupBy { get; set; } public Select Select { get; set; } public HashSet Fields { get; set; } public int Limit { get; set; } = int.MaxValue; public int Offset { get; set; } public bool ForUpdate { get; set; } public QueryPlan(string collection) { Collection = collection; } public BasePipe GetPipe(TransactionService transaction, Snapshot snapshot, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount) { if (GroupBy == null) { return new QueryPipe(transaction, GetLookup(snapshot, pragmas, maxItemsCount), tempDisk, pragmas, maxItemsCount); } return new GroupByPipe(transaction, GetLookup(snapshot, pragmas, maxItemsCount), tempDisk, pragmas, maxItemsCount); } public IDocumentLookup GetLookup(Snapshot snapshot, EnginePragmas pragmas, uint maxItemsCount) { DataService data = new DataService(snapshot, maxItemsCount); IndexService indexer = new IndexService(snapshot, pragmas.Collation, maxItemsCount); IDocumentLookup documentLookup = Index as IDocumentLookup; if (documentLookup == null) { documentLookup = ((!IsIndexKeyOnly) ? ((IDocumentLookup)new DatafileLookup(data, pragmas.UtcDate, Fields)) : ((IDocumentLookup)new IndexLookup(indexer, Fields.Single()))); } return documentLookup; } public BsonDocument GetExecutionPlan() { BsonDocument bsonDocument = new BsonDocument { ["collection"] = Collection, ["snaphost"] = (ForUpdate ? "write" : "read"), ["pipe"] = ((GroupBy == null) ? "queryPipe" : "groupByPipe") }; bsonDocument["index"] = new BsonDocument { ["name"] = Index.Name, ["expr"] = IndexExpression, ["order"] = Index.Order, ["mode"] = Index.ToString(), ["cost"] = (int)IndexCost }; bsonDocument["lookup"] = new BsonDocument { ["loader"] = ((Index is IndexVirtual) ? "virtual" : (IsIndexKeyOnly ? "index" : "document")), ["fields"] = ((Fields.Count == 0) ? new BsonValue("$") : new BsonArray(Fields.Select((string x) => new BsonValue(x)))) }; if (IncludeBefore.Count > 0) { bsonDocument["includeBefore"] = new BsonArray(IncludeBefore.Select((BsonExpression x) => new BsonValue(x.Source))); } if (Filters.Count > 0) { bsonDocument["filters"] = new BsonArray(Filters.Select((BsonExpression x) => new BsonValue(x.Source))); } if (OrderBy != null) { bsonDocument["orderBy"] = new BsonDocument { ["expr"] = OrderBy.Expression.Source, ["order"] = OrderBy.Order }; } if (Limit != int.MaxValue) { bsonDocument["limit"] = Limit; } if (Offset != 0) { bsonDocument["offset"] = Offset; } if (IncludeAfter.Count > 0) { bsonDocument["includeAfter"] = new BsonArray(IncludeAfter.Select((BsonExpression x) => new BsonValue(x.Source))); } if (GroupBy != null) { bsonDocument["groupBy"] = new BsonDocument { ["expr"] = GroupBy.Expression.Source, ["having"] = GroupBy.Having?.Source, ["select"] = GroupBy.Select?.Source }; } else { bsonDocument["select"] = new BsonDocument { ["expr"] = Select.Expression.Source, ["all"] = Select.All }; } return bsonDocument; } } internal class Select { public BsonExpression Expression { get; } public bool All { get; } public Select(BsonExpression expression, bool all) { Expression = expression; All = all; } } internal class CollectionService { private readonly HeaderPage _header; private readonly DiskService _disk; private readonly Snapshot _snapshot; private readonly TransactionPages _transPages; public CollectionService(HeaderPage header, DiskService disk, Snapshot snapshot, TransactionPages transPages) { _header = header; _disk = disk; _snapshot = snapshot; _transPages = transPages; } public static void CheckName(string name, HeaderPage header) { if (Encoding.UTF8.GetByteCount(name) > header.GetAvailableCollectionSpace()) { throw LiteException.InvalidCollectionName(name, "There is no space in header this collection name"); } if (!name.IsWord()) { throw LiteException.InvalidCollectionName(name, "Use only [a-Z$_]"); } if (name.StartsWith("$")) { throw LiteException.InvalidCollectionName(name, "Collection can't starts with `$` (reserved for system collections)"); } } public bool Get(string name, bool addIfNotExists, ref CollectionPage collectionPage) { uint collectionPageID = _header.GetCollectionPageID(name); if (collectionPageID != uint.MaxValue) { collectionPage = _snapshot.GetPage(collectionPageID); return false; } if (addIfNotExists) { Add(name, ref collectionPage); return true; } return false; } private void Add(string name, ref CollectionPage collectionPage) { CheckName(name, _header); collectionPage = _snapshot.NewPage(); uint pageID = collectionPage.PageID; _transPages.Commit += delegate(HeaderPage h) { h.InsertCollection(name, pageID); }; new IndexService(_snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT).CreateIndex("_id", "$._id", unique: true); } } internal class DataService { public const int MAX_DATA_BYTES_PER_PAGE = 8150; private readonly Snapshot _snapshot; private readonly uint _maxItemsCount; public DataService(Snapshot snapshot, uint maxItemsCount) { _snapshot = snapshot; _maxItemsCount = maxItemsCount; } public PageAddress Insert(BsonDocument doc) { int bytesLeft = doc.GetBytesCount(recalc: true); if (bytesLeft > 16683050) { throw new LiteException(0, "Document size exceed {0} limit", 16683050); } PageAddress firstBlock = PageAddress.Empty; using (BufferWriter bufferWriter = new BufferWriter(source())) { bufferWriter.WriteDocument(doc, recalc: false); bufferWriter.Consume(); } return firstBlock; IEnumerable source() { int blockIndex = 0; DataBlock dataBlock2 = null; int bytesToCopy; for (; bytesLeft > 0; bytesLeft -= bytesToCopy) { bytesToCopy = Math.Min(bytesLeft, 8150); DataPage freeDataPage = _snapshot.GetFreeDataPage(bytesToCopy + 6); DataBlock dataBlock = freeDataPage.InsertBlock(bytesToCopy, blockIndex++ > 0); dataBlock2?.SetNextBlock(dataBlock.Position); if (firstBlock.IsEmpty) { firstBlock = dataBlock.Position; } _snapshot.AddOrRemoveFreeDataList(freeDataPage); yield return dataBlock.Buffer; dataBlock2 = dataBlock; } } } public void Update(CollectionPage col, PageAddress blockAddress, BsonDocument doc) { int bytesLeft = doc.GetBytesCount(recalc: true); if (bytesLeft > 16683050) { throw new LiteException(0, "Document size exceed {0} limit", 16683050); } DataBlock lastBlock = null; PageAddress updateAddress = blockAddress; using (BufferWriter bufferWriter = new BufferWriter(source())) { bufferWriter.WriteDocument(doc, recalc: false); bufferWriter.Consume(); } IEnumerable source() { int bytesToCopy; for (; bytesLeft > 0; bytesLeft -= bytesToCopy) { if (!updateAddress.IsEmpty) { DataPage page = _snapshot.GetPage(updateAddress.PageID); DataBlock block = page.GetBlock(updateAddress.Index); bytesToCopy = Math.Min(bytesLeft, page.FreeBytes + block.Buffer.Count); DataBlock updateBlock2 = page.UpdateBlock(block, bytesToCopy); _snapshot.AddOrRemoveFreeDataList(page); yield return updateBlock2.Buffer; lastBlock = updateBlock2; updateAddress = updateBlock2.NextBlock; } else { bytesToCopy = Math.Min(bytesLeft, 8150); DataPage freeDataPage = _snapshot.GetFreeDataPage(bytesToCopy + 6); DataBlock updateBlock2 = freeDataPage.InsertBlock(bytesToCopy, extend: true); if (lastBlock != null) { lastBlock.SetNextBlock(updateBlock2.Position); } _snapshot.AddOrRemoveFreeDataList(freeDataPage); yield return updateBlock2.Buffer; lastBlock = updateBlock2; } } if (!lastBlock.NextBlock.IsEmpty) { PageAddress nextBlock = lastBlock.NextBlock; lastBlock.SetNextBlock(PageAddress.Empty); Delete(nextBlock); } } } public IEnumerable Read(PageAddress address) { uint counter = 0u; while (address != PageAddress.Empty) { Constants.ENSURE(counter++ < _maxItemsCount, "Detected loop in data Read({0})", address); DataPage page = _snapshot.GetPage(address.PageID); DataBlock block = page.GetBlock(address.Index); yield return block.Buffer; address = block.NextBlock; } } public void Delete(PageAddress blockAddress) { while (blockAddress != PageAddress.Empty) { DataPage page = _snapshot.GetPage(blockAddress.PageID); DataBlock block = page.GetBlock(blockAddress.Index); page.DeleteBlock(blockAddress.Index); _snapshot.AddOrRemoveFreeDataList(page); blockAddress = block.NextBlock; } } } internal class IndexService { private readonly Snapshot _snapshot; private readonly Collation _collation; private readonly uint _maxItemsCount; public Collation Collation => _collation; public IndexService(Snapshot snapshot, Collation collation, uint maxItemsCount) { _snapshot = snapshot; _collation = collation; _maxItemsCount = maxItemsCount; } public CollectionIndex CreateIndex(string name, string expr, bool unique) { int keyLength; int nodeLength = IndexNode.GetNodeLength(32, BsonValue.MinValue, out keyLength); IndexPage indexPage = _snapshot.NewPage(); CollectionIndex collectionIndex = _snapshot.CollectionPage.InsertCollectionIndex(name, expr, unique); IndexNode indexNode = indexPage.InsertIndexNode(collectionIndex.Slot, 32, BsonValue.MinValue, PageAddress.Empty, nodeLength); IndexNode indexNode2 = indexPage.InsertIndexNode(collectionIndex.Slot, 32, BsonValue.MaxValue, PageAddress.Empty, nodeLength); indexNode.SetNext(0, indexNode2.Position); indexNode2.SetPrev(0, indexNode.Position); collectionIndex.FreeIndexPageList = indexPage.PageID; indexPage.PageListSlot = 0; collectionIndex.Head = indexNode.Position; collectionIndex.Tail = indexNode2.Position; return collectionIndex; } public IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, IndexNode last) { if (key.IsMaxValue || key.IsMinValue) { throw LiteException.InvalidIndexKey("BsonValue MaxValue/MinValue are not supported as index key"); } byte insertLevels = Flip(); return AddNode(index, key, dataBlock, insertLevels, last); } private IndexNode AddNode(CollectionIndex index, BsonValue key, PageAddress dataBlock, byte insertLevels, IndexNode last) { int keyLength; int nodeLength = IndexNode.GetNodeLength(insertLevels, key, out keyLength); if (keyLength > 1023) { throw LiteException.InvalidIndexKey($"Index key must be less than {1023} bytes."); } IndexNode indexNode = _snapshot.GetFreeIndexPage(nodeLength, ref index.FreeIndexPageList).InsertIndexNode(index.Slot, insertLevels, key, dataBlock, nodeLength); IndexNode indexNode2 = GetNode(index.Head); uint num = 0u; for (int num2 = 31; num2 >= 0; num2--) { PageAddress pageAddress = indexNode2.Next[num2]; while (!pageAddress.IsEmpty && pageAddress != index.Tail) { Constants.ENSURE(num++ < _maxItemsCount, "Detected loop in AddNode({0})", indexNode.Position); IndexNode node = GetNode(pageAddress); int num3 = node.Key.CompareTo(key, _collation); if (num3 == 0 && index.Unique) { throw LiteException.IndexDuplicateKey(index.Name, key); } if (num3 == 1) { break; } indexNode2 = node; pageAddress = node.Next[num2]; } if (num2 <= insertLevels - 1) { PageAddress position = indexNode2.Position; PageAddress value = indexNode2.Next[num2]; if (value.IsEmpty) { value = index.Tail; } indexNode.SetNext((byte)num2, value); indexNode.SetPrev((byte)num2, position); indexNode2.SetNext((byte)num2, indexNode.Position); pageAddress = indexNode.Next[num2]; GetNode(pageAddress).SetPrev((byte)num2, indexNode.Position); } } if (last != null) { Constants.ENSURE(last.NextNode == PageAddress.Empty, "last index node must point to null"); last = GetNode(last.Position); last.SetNextNode(indexNode.Position); } _snapshot.AddOrRemoveFreeIndexList(indexNode.Page, ref index.FreeIndexPageList); return indexNode; } public byte Flip() { byte b = 1; int num = Randomizer.Next(); while ((num & 1) == 1) { b++; if (b == 32) { break; } num >>= 1; } return b; } public IndexNode GetNode(PageAddress address) { if (address.PageID == uint.MaxValue) { return null; } return _snapshot.GetPage(address.PageID).GetIndexNode(address.Index); } public IEnumerable GetNodeList(PageAddress nodeAddress) { IndexNode node = GetNode(nodeAddress); uint counter = 0u; while (node != null) { Constants.ENSURE(counter++ < _maxItemsCount, "Detected loop in GetNodeList({0})", nodeAddress); yield return node; node = GetNode(node.NextNode); } } public void DeleteAll(PageAddress pkAddress) { IndexNode node = GetNode(pkAddress); CollectionIndex[] collectionIndexesSlots = _snapshot.CollectionPage.GetCollectionIndexesSlots(); uint num = 0u; while (node != null) { Constants.ENSURE(num++ < _maxItemsCount, "Detected loop in DeleteAll({0})", pkAddress); DeleteSingleNode(node, collectionIndexesSlots[node.Slot]); node = GetNode(node.NextNode); } } public IndexNode DeleteList(PageAddress pkAddress, HashSet toDelete) { IndexNode indexNode = GetNode(pkAddress); IndexNode node = GetNode(indexNode.NextNode); CollectionIndex[] collectionIndexesSlots = _snapshot.CollectionPage.GetCollectionIndexesSlots(); uint num = 0u; while (node != null) { Constants.ENSURE(num++ < _maxItemsCount, "Detected loop in DeleteList({0})", pkAddress); if (toDelete.Contains(node.Position)) { DeleteSingleNode(node, collectionIndexesSlots[node.Slot]); indexNode.SetNextNode(node.NextNode); } else { indexNode = node; } node = GetNode(node.NextNode); } return indexNode; } private void DeleteSingleNode(IndexNode node, CollectionIndex index) { for (int num = node.Levels - 1; num >= 0; num--) { IndexNode node2 = GetNode(node.Prev[num]); IndexNode node3 = GetNode(node.Next[num]); node2?.SetNext((byte)num, node.Next[num]); node3?.SetPrev((byte)num, node.Prev[num]); } node.Page.DeleteIndexNode(node.Position.Index); _snapshot.AddOrRemoveFreeIndexList(node.Page, ref index.FreeIndexPageList); } public void DropIndex(CollectionIndex index) { byte slot = index.Slot; CollectionIndex pK = _snapshot.CollectionPage.PK; foreach (IndexNode item in FindAll(pK, 1)) { PageAddress nextNode = item.NextNode; IndexNode indexNode = item; while (nextNode != PageAddress.Empty) { IndexNode node = GetNode(nextNode); if (node.Slot == slot) { node.Page.DeleteIndexNode(node.Position.Index); indexNode.SetNextNode(node.NextNode); } else { indexNode = node; } nextNode = node.NextNode; } } GetNode(index.Head).Page.DeleteIndexNode(index.Head.Index); GetNode(index.Tail).Page.DeleteIndexNode(index.Tail.Index); } public IEnumerable FindAll(CollectionIndex index, int order) { IndexNode cur = ((order == 1) ? GetNode(index.Head) : GetNode(index.Tail)); uint counter = 0u; while (!cur.GetNextPrev(0, order).IsEmpty) { Constants.ENSURE(counter++ < _maxItemsCount, "Detected loop in FindAll({0})", index.Name); cur = GetNode(cur.GetNextPrev(0, order)); if (cur.Key.IsMinValue || cur.Key.IsMaxValue) { break; } yield return cur; } } public IndexNode Find(CollectionIndex index, BsonValue value, bool sibling, int order) { IndexNode indexNode = ((order == 1) ? GetNode(index.Head) : GetNode(index.Tail)); uint num = 0u; for (int num2 = 31; num2 >= 0; num2--) { PageAddress nextPrev = indexNode.GetNextPrev((byte)num2, order); while (!nextPrev.IsEmpty) { Constants.ENSURE(num++ < _maxItemsCount, "Detected loop in Find({0}, {1})", index.Name, value); IndexNode node = GetNode(nextPrev); int num3 = node.Key.CompareTo(value, _collation); if (num3 == order && (num2 > 0 || !sibling)) { break; } if (num3 == order && num2 == 0 && sibling) { if (!node.Key.IsMinValue && !node.Key.IsMaxValue) { return node; } return null; } if (num3 == 0) { return node; } indexNode = node; nextPrev = node.GetNextPrev((byte)num2, order); } } return null; } } internal class LockService : IDisposable { private readonly EnginePragmas _pragmas; private readonly ReaderWriterLockSlim _transaction = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion); private readonly ConcurrentDictionary _collections = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); public bool IsInTransaction { get { if (!_transaction.IsReadLockHeld) { return _transaction.IsWriteLockHeld; } return true; } } public int TransactionsCount => _transaction.CurrentReadCount; internal LockService(EnginePragmas pragmas) { _pragmas = pragmas; } public void EnterTransaction() { if (_transaction.IsWriteLockHeld || _transaction.TryEnterReadLock(_pragmas.Timeout)) { return; } throw LiteException.LockTimeout("transaction", _pragmas.Timeout); } public void ExitTransaction() { if (_transaction.IsWriteLockHeld || !_transaction.IsReadLockHeld) { return; } try { _transaction.ExitReadLock(); } catch { } } public void EnterLock(string collectionName) { Constants.ENSURE(_transaction.IsReadLockHeld || _transaction.IsWriteLockHeld, "Use EnterTransaction() before EnterLock(name)"); if (!Monitor.TryEnter(_collections.GetOrAdd(collectionName, (string s) => new object()), _pragmas.Timeout)) { throw LiteException.LockTimeout("write", collectionName, _pragmas.Timeout); } } public void ExitLock(string collectionName) { if (!_collections.TryGetValue(collectionName, out var value)) { throw LiteException.CollectionLockerNotFound(collectionName); } Monitor.Exit(value); } public bool EnterExclusive() { if (_transaction.IsWriteLockHeld) { return false; } if (!_transaction.TryEnterWriteLock(_pragmas.Timeout)) { throw LiteException.LockTimeout("exclusive", _pragmas.Timeout); } return true; } public bool TryEnterExclusive(out bool mustExit) { if (_transaction.IsWriteLockHeld) { mustExit = false; return true; } if (_transaction.IsReadLockHeld || _transaction.CurrentReadCount > 0) { mustExit = false; return false; } if (!_transaction.TryEnterWriteLock(10)) { mustExit = false; return false; } Constants.ENSURE(_transaction.RecursiveReadCount == 0, "must have no other transaction here"); mustExit = true; return true; } public void ExitExclusive() { _transaction.ExitWriteLock(); } public void Dispose() { try { _transaction.Dispose(); } catch (SynchronizationLockException) { } } } internal class RebuildService { private readonly EngineSettings _settings; private readonly int _fileVersion; public RebuildService(EngineSettings settings) { _settings = settings; if (FileReaderV7.IsVersion(ReadFirstBytes(useAesStream: false))) { _fileVersion = 7; return; } byte[] buffer = ReadFirstBytes(); if (!FileReaderV8.IsVersion(buffer)) { throw LiteException.InvalidDatabase(); } _fileVersion = 8; } public long Rebuild(RebuildOptions options) { string backupFilename = FileHelper.GetSuffixFile(_settings.Filename, "-backup"); string suffixFile = FileHelper.GetSuffixFile(FileHelper.GetLogFile(_settings.Filename), "-backup"); string suffixFile2 = FileHelper.GetSuffixFile(_settings.Filename); IFileReader fileReader2; if (_fileVersion != 7) { IFileReader fileReader = new FileReaderV8(_settings, options.Errors); fileReader2 = fileReader; } else { IFileReader fileReader = new FileReaderV7(_settings); fileReader2 = fileReader; } using (IFileReader fileReader3 = fileReader2) { fileReader3.Open(); using LiteEngine liteEngine = new LiteEngine(new EngineSettings { Filename = suffixFile2, Collation = options.Collation, Password = options.Password }); liteEngine.Pragma("CHECKPOINT", 0); liteEngine.RebuildContent(fileReader3); if (options.IncludeErrorReport && options.Errors.Count > 0) { IEnumerable errorReport = options.GetErrorReport(); liteEngine.Insert("_rebuild_errors", errorReport, BsonAutoId.Int32); } IDictionary pragmas = fileReader3.GetPragmas(); liteEngine.Pragma("CHECKPOINT", pragmas["CHECKPOINT"]); liteEngine.Pragma("TIMEOUT", pragmas["TIMEOUT"]); liteEngine.Pragma("LIMIT_SIZE", pragmas["LIMIT_SIZE"]); liteEngine.Pragma("UTC_DATE", pragmas["UTC_DATE"]); liteEngine.Pragma("USER_VERSION", pragmas["USER_VERSION"]); liteEngine.Checkpoint(); } string logFile = FileHelper.GetLogFile(_settings.Filename); if (File.Exists(logFile)) { File.Move(logFile, suffixFile); } FileHelper.Exec(5, delegate { File.Move(_settings.Filename, backupFilename); }); File.Move(suffixFile2, _settings.Filename); return new FileInfo(backupFilename).Length - new FileInfo(_settings.Filename).Length; } private byte[] ReadFirstBytes(bool useAesStream = true) { byte[] array = new byte[16384]; using Stream stream = _settings.CreateDataFactory(useAesStream).GetStream(canWrite: false, sequencial: true); stream.Position = 0L; stream.Read(array, 0, array.Length); return array; } } internal class Snapshot : IDisposable { private readonly HeaderPage _header; private readonly LockService _locker; private readonly DiskReader _reader; private readonly DiskService _disk; private readonly WalIndexService _walIndex; private readonly uint _transactionID; private readonly TransactionPages _transPages; private readonly int _readVersion; private readonly LockMode _mode; private readonly string _collectionName; private readonly CollectionPage _collectionPage; private readonly Dictionary _localPages = new Dictionary(); private bool _disposed; public LockMode Mode => _mode; public string CollectionName => _collectionName; public CollectionPage CollectionPage => _collectionPage; public ICollection LocalPages => _localPages.Values; public int ReadVersion => _readVersion; public Snapshot(LockMode mode, string collectionName, HeaderPage header, uint transactionID, TransactionPages transPages, LockService locker, WalIndexService walIndex, DiskReader reader, DiskService disk, bool addIfNotExists) { _mode = mode; _collectionName = collectionName; _header = header; _transactionID = transactionID; _transPages = transPages; _locker = locker; _walIndex = walIndex; _reader = reader; _disk = disk; if (mode == LockMode.Write) { _locker.EnterLock(_collectionName); } _readVersion = _walIndex.CurrentReadVersion; new CollectionService(_header, _disk, this, _transPages).Get(_collectionName, addIfNotExists, ref _collectionPage); if (_collectionPage != null) { _localPages.Remove(_collectionPage.PageID); } } public IEnumerable GetWritablePages(bool dirty, bool includeCollectionPage) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); if (_mode == LockMode.Read) { yield break; } foreach (BasePage item in _localPages.Values.Where((BasePage x) => x.IsDirty == dirty)) { Constants.ENSURE(item.PageType != PageType.Header && item.PageType != PageType.Collection, "local cache cann't contains this page type"); yield return item; } if (includeCollectionPage && _collectionPage != null && _collectionPage.IsDirty == dirty) { yield return _collectionPage; } } public void Clear() { Constants.ENSURE(!_disposed, "the snapshot is disposed"); if (_mode == LockMode.Read) { foreach (BasePage value in _localPages.Values) { value.Buffer.Release(); } } _localPages.Clear(); } public void Dispose() { if (!_disposed) { Clear(); _disposed = true; if (_mode == LockMode.Read && _collectionPage != null) { _collectionPage.Buffer.Release(); } if (_mode == LockMode.Write) { _locker.ExitLock(_collectionName); } } } public T GetPage(uint pageID, bool useLatestVersion = false) where T : BasePage { FileOrigin origin; long position; int walVersion; return GetPage(pageID, out origin, out position, out walVersion, useLatestVersion); } public T GetPage(uint pageID, out FileOrigin origin, out long position, out int walVersion, bool useLatestVersion = false) where T : BasePage { Constants.ENSURE(!_disposed, "the snapshot is disposed"); Constants.ENSURE(pageID <= _header.LastPageID, "request page must be less or equals lastest page in data file"); if (pageID == 0) { origin = FileOrigin.None; position = 0L; walVersion = 0; return (T)(BasePage)_header; } if (_localPages.TryGetValue(pageID, out var value)) { origin = FileOrigin.None; position = 0L; walVersion = 0; return (T)value; } value = ReadPage(pageID, out origin, out position, out walVersion, useLatestVersion); _localPages[pageID] = value; _transPages.TransactionSize++; return (T)value; } private T ReadPage(uint pageID, out FileOrigin origin, out long position, out int walVersion, bool useLatestVersion = false) where T : BasePage { if (_transPages.DirtyPages.TryGetValue(pageID, out var value)) { T val = BasePage.ReadPage(_reader.ReadPage(value.Position, _mode == LockMode.Write, FileOrigin.Log)); origin = FileOrigin.Log; position = value.Position; walVersion = _readVersion; Constants.ENSURE(val.TransactionID == _transactionID, "this page must came from same transaction"); return val; } long pageIndex = _walIndex.GetPageIndex(pageID, useLatestVersion ? int.MaxValue : _readVersion, out walVersion); if (pageIndex != long.MaxValue) { T val2 = BasePage.ReadPage(_reader.ReadPage(pageIndex, _mode == LockMode.Write, FileOrigin.Log)); val2.TransactionID = 0u; val2.IsConfirmed = false; origin = FileOrigin.Log; position = pageIndex; return val2; } long pagePosition = BasePage.GetPagePosition(pageID); T val3 = BasePage.ReadPage(_reader.ReadPage(pagePosition, _mode == LockMode.Write, FileOrigin.Data)); origin = FileOrigin.Data; position = pagePosition; Constants.ENSURE(!val3.IsConfirmed || val3.TransactionID != 0, "page are not header-clear in data file"); return val3; } public DataPage GetFreeDataPage(int bytesLength) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); int num = bytesLength + 4; for (int num2 = DataPage.GetMinimumIndexSlot(num); num2 >= 0; num2--) { uint num3 = _collectionPage.FreeDataPageList[num2]; if (num3 != uint.MaxValue) { DataPage page = GetPage(num3); Constants.ENSURE(page.PageListSlot == num2, "stored slot must be same as called"); Constants.ENSURE(page.FreeBytes >= num, "ensure selected page has space enough for this content"); page.IsDirty = true; return page; } } return NewPage(); } public IndexPage GetFreeIndexPage(int bytesLength, ref uint freeIndexPageList) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); IndexPage indexPage; if (freeIndexPageList == uint.MaxValue) { indexPage = NewPage(); } else { indexPage = GetPage(freeIndexPageList); Constants.ENSURE(indexPage.FreeBytes > bytesLength, "this page shout be space enouth for this new node"); Constants.ENSURE(indexPage.PageListSlot == 0, "this page should be in slot #0"); } return indexPage; } public T NewPage() where T : BasePage { Constants.ENSURE(!_disposed, "the snapshot is disposed"); Constants.ENSURE(_collectionPage == null, typeof(T) == typeof(CollectionPage), "if no collection page defined yet, must be first request"); Constants.ENSURE(typeof(T) == typeof(CollectionPage), _collectionPage == null, "there is no new collection page if page already exists"); uint num = 0u; PageBuffer buffer; lock (_header) { if (_header.FreeEmptyPageList != uint.MaxValue) { BasePage page = GetPage(_header.FreeEmptyPageList, useLatestVersion: true); Constants.ENSURE(page.PageType == PageType.Empty, "empty page must be defined as empty type"); _header.FreeEmptyPageList = page.NextPageID; page.NextPageID = uint.MaxValue; num = page.PageID; buffer = page.Buffer; } else { if ((_header.LastPageID + 1) * 8192 > _header.Pragmas.LimitSize) { throw new LiteException(0, "Maximum data file size has been reached: " + FileHelper.FormatFileSize(_header.Pragmas.LimitSize)); } PageBuffer savepoint = _header.Savepoint(); try { num = ++_header.LastPageID; buffer = _reader.NewPage(); } catch { _header.Restore(savepoint); throw; } } _transPages.NewPages.Add(num); } T val = BasePage.CreatePage(buffer, num); if (val.PageType != PageType.Collection) { _localPages[num] = val; } val.ColID = _collectionPage?.PageID ?? val.PageID; val.IsDirty = true; _transPages.TransactionSize++; return val; } public void AddOrRemoveFreeDataList(DataPage page) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); byte b = DataPage.FreeIndexSlot(page.FreeBytes); byte pageListSlot = page.PageListSlot; if (b != pageListSlot || page.ItemsCount <= 0) { if (pageListSlot != byte.MaxValue) { RemoveFreeList(page, ref _collectionPage.FreeDataPageList[pageListSlot]); } if (page.ItemsCount == 0) { DeletePage(page); return; } AddFreeList(page, ref _collectionPage.FreeDataPageList[b]); page.PageListSlot = b; } } public void AddOrRemoveFreeIndexList(IndexPage page, ref uint startPageID) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); byte b = IndexPage.FreeIndexSlot(page.FreeBytes); bool flag = page.PageListSlot == 0; bool flag2 = b == 0; if (page.ItemsCount == 0) { if (flag) { RemoveFreeList(page, ref startPageID); } DeletePage(page); return; } if (flag && !flag2) { RemoveFreeList(page, ref startPageID); } else if (!flag && flag2) { AddFreeList(page, ref startPageID); } page.PageListSlot = b; page.IsDirty = true; } private void AddFreeList(T page, ref uint startPageID) where T : BasePage { Constants.ENSURE(page.PrevPageID == uint.MaxValue && page.NextPageID == uint.MaxValue, "only non-linked page can be added in linked list"); if (startPageID != uint.MaxValue) { T page2 = GetPage(startPageID); page2.PrevPageID = page.PageID; page2.IsDirty = true; } page.PrevPageID = uint.MaxValue; page.NextPageID = startPageID; page.IsDirty = true; Constants.ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index, "only data/index pages must be first on free stack"); startPageID = page.PageID; _collectionPage.IsDirty = true; } private void RemoveFreeList(T page, ref uint startPageID) where T : BasePage { if (page.PrevPageID != uint.MaxValue) { T page2 = GetPage(page.PrevPageID); page2.NextPageID = page.NextPageID; page2.IsDirty = true; } if (page.NextPageID != uint.MaxValue) { T page3 = GetPage(page.NextPageID); page3.PrevPageID = page.PrevPageID; page3.IsDirty = true; } if (startPageID == page.PageID) { startPageID = page.NextPageID; Constants.ENSURE(page.NextPageID == uint.MaxValue || GetPage(page.NextPageID).PageType != PageType.Empty, "first page on free stack must be non empty page"); _collectionPage.IsDirty = true; } uint prevPageID = (page.NextPageID = uint.MaxValue); page.PrevPageID = prevPageID; page.IsDirty = true; } private void DeletePage(T page) where T : BasePage { Constants.ENSURE(page.PrevPageID == uint.MaxValue && page.NextPageID == uint.MaxValue, "before delete a page, no linked list with any another page"); Constants.ENSURE(page.ItemsCount == 0 && page.UsedBytes == 0 && page.HighestIndex == byte.MaxValue && page.FragmentedBytes == 0, "no items on page when delete this page"); Constants.ENSURE(page.PageType == PageType.Data || page.PageType == PageType.Index, "only data/index page can be deleted"); page.MarkAsEmtpy(); if (_transPages.FirstDeletedPageID == uint.MaxValue) { Constants.ENSURE(_transPages.DeletedPages == 0, "if has no firstDeletedPageID must has deleted pages"); _transPages.FirstDeletedPageID = page.PageID; _transPages.LastDeletedPageID = page.PageID; } else { Constants.ENSURE(_transPages.DeletedPages > 0, "must have at least 1 deleted page"); page.NextPageID = _transPages.FirstDeletedPageID; _transPages.FirstDeletedPageID = page.PageID; } _transPages.DeletedPages++; } public void DropCollection(Action safePoint) { Constants.ENSURE(!_disposed, "the snapshot is disposed"); IndexService indexService = new IndexService(this, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT); _transPages.FirstDeletedPageID = _collectionPage.PageID; _transPages.LastDeletedPageID = _collectionPage.PageID; _collectionPage.MarkAsEmtpy(); _transPages.DeletedPages = 1; HashSet hashSet = new HashSet(); foreach (CollectionIndex collectionIndex in _collectionPage.GetCollectionIndexes()) { hashSet.Add(collectionIndex.Head.PageID); foreach (IndexNode item in indexService.FindAll(collectionIndex, 1)) { hashSet.Add(item.Page.PageID); safePoint(); } } foreach (uint item2 in hashSet) { IndexPage page = GetPage(item2); page.MarkAsEmtpy(); page.NextPageID = _transPages.FirstDeletedPageID; _transPages.FirstDeletedPageID = page.PageID; _transPages.DeletedPages++; safePoint(); } uint[] freeDataPageList = _collectionPage.FreeDataPageList; for (int i = 0; i < freeDataPageList.Length; i++) { uint num = freeDataPageList[i]; while (num != uint.MaxValue) { DataPage page2 = GetPage(num); num = page2.NextPageID; page2.MarkAsEmtpy(); page2.NextPageID = _transPages.FirstDeletedPageID; _transPages.FirstDeletedPageID = page2.PageID; _transPages.DeletedPages++; safePoint(); } } _transPages.Commit += delegate(HeaderPage h) { h.DeleteCollection(_collectionName); }; } public override string ToString() { return $"{_collectionName} (pages: {_localPages.Count})"; } } internal class TransactionMonitor : IDisposable { private readonly Dictionary _transactions = new Dictionary(); private readonly ThreadLocal _slot = new ThreadLocal(); private readonly HeaderPage _header; private readonly LockService _locker; private readonly DiskService _disk; private readonly WalIndexService _walIndex; private int _freePages; private readonly int _initialSize; public ICollection Transactions => _transactions.Values; public int FreePages => _freePages; public int InitialSize => _initialSize; public TransactionMonitor(HeaderPage header, LockService locker, DiskService disk, WalIndexService walIndex) { _header = header; _locker = locker; _disk = disk; _walIndex = walIndex; _freePages = 100000; _initialSize = 1000; } public TransactionService GetTransaction(bool create, bool queryOnly, out bool isNew) { TransactionService transactionService = _slot.Value; if (create && transactionService == null) { isNew = true; bool flag; lock (_transactions) { if (_transactions.Count >= 100) { throw new LiteException(0, "Maximum number of transactions reached"); } int initialSize = GetInitialSize(); flag = _transactions.Values.Any((TransactionService x) => x.ThreadID == Environment.CurrentManagedThreadId); transactionService = new TransactionService(_header, _locker, _disk, _walIndex, initialSize, this, queryOnly); _transactions[transactionService.TransactionID] = transactionService; } if (!flag) { try { _locker.EnterTransaction(); } catch { transactionService.Dispose(); lock (_transactions) { _freePages += transactionService.MaxTransactionSize; _transactions.Remove(transactionService.TransactionID); } throw; } } if (!queryOnly) { _slot.Value = transactionService; } } else { isNew = false; } return transactionService; } public void ReleaseTransaction(TransactionService transaction) { transaction.Dispose(); bool flag; lock (_transactions) { _transactions.Remove(transaction.TransactionID); _freePages += transaction.MaxTransactionSize; flag = _transactions.Values.Any((TransactionService x) => x.ThreadID == Environment.CurrentManagedThreadId); } if (!flag) { _locker.ExitTransaction(); } if (!transaction.QueryOnly) { Constants.ENSURE(_slot.Value == transaction, "current thread must contains transaction parameter"); _slot.Value = null; } } public TransactionService GetThreadTransaction() { lock (_transactions) { return _slot.Value ?? _transactions.Values.FirstOrDefault((TransactionService x) => x.ThreadID == Environment.CurrentManagedThreadId); } } private int GetInitialSize() { if (_freePages >= _initialSize) { _freePages -= _initialSize; return _initialSize; } int num = 0; foreach (TransactionService value in _transactions.Values) { int num2 = value.MaxTransactionSize / _initialSize; value.MaxTransactionSize -= num2; num += num2; } return num; } private bool TryExtend(TransactionService trans) { lock (_transactions) { if (_freePages >= _initialSize) { trans.MaxTransactionSize += _initialSize; _freePages -= _initialSize; return true; } return false; } } public bool CheckSafepoint(TransactionService trans) { if (trans.Pages.TransactionSize >= trans.MaxTransactionSize) { return !TryExtend(trans); } return false; } public void Dispose() { if (_transactions.Count <= 0) { return; } foreach (TransactionService value in _transactions.Values) { value.Dispose(); } _transactions.Clear(); } } internal class TransactionService : IDisposable { private readonly HeaderPage _header; private readonly LockService _locker; private readonly DiskService _disk; private readonly DiskReader _reader; private readonly WalIndexService _walIndex; private readonly TransactionMonitor _monitor; private readonly Dictionary _snapshots = new Dictionary(StringComparer.OrdinalIgnoreCase); private readonly TransactionPages _transPages = new TransactionPages(); private readonly int _threadID = Environment.CurrentManagedThreadId; private readonly uint _transactionID; private readonly DateTime _startTime; private LockMode _mode; private TransactionState _state; public int ThreadID => _threadID; public uint TransactionID => _transactionID; public TransactionState State => _state; public LockMode Mode => _mode; public TransactionPages Pages => _transPages; public DateTime StartTime => _startTime; public IEnumerable Snapshots => _snapshots.Values; public bool QueryOnly { get; } public int MaxTransactionSize { get; set; } public List OpenCursors { get; } = new List(); public bool ExplicitTransaction { get; set; } public TransactionService(HeaderPage header, LockService locker, DiskService disk, WalIndexService walIndex, int maxTransactionSize, TransactionMonitor monitor, bool queryOnly) { _header = header; _locker = locker; _disk = disk; _walIndex = walIndex; _monitor = monitor; QueryOnly = queryOnly; MaxTransactionSize = maxTransactionSize; _transactionID = walIndex.NextTransactionID(); _startTime = DateTime.UtcNow; _reader = _disk.GetReader(); } ~TransactionService() { Dispose(dispose: false); } public Snapshot CreateSnapshot(LockMode mode, string collection, bool addIfNotExists) { Constants.ENSURE(_state == TransactionState.Active, "transaction must be active to create new snapshot"); if (!_snapshots.TryGetValue(collection, out var value)) { value = (_snapshots[collection] = create()); } else if ((mode == LockMode.Write && value.Mode == LockMode.Read) || (addIfNotExists && value.CollectionPage == null)) { value.Dispose(); _snapshots.Remove(collection); value = (_snapshots[collection] = create()); } if (mode == LockMode.Write) { _mode = LockMode.Write; } return value; Snapshot create() { return new Snapshot(mode, collection, _header, _transactionID, _transPages, _locker, _walIndex, _reader, _disk, addIfNotExists); } } public void Safepoint() { if (_state != 0) { throw new LiteException(0, "This transaction are invalid state"); } if (!_monitor.CheckSafepoint(this)) { return; } if (_mode == LockMode.Write) { PersistDirtyPages(commit: false); } foreach (Snapshot value in _snapshots.Values) { value.Clear(); } _transPages.TransactionSize = 0; } private int PersistDirtyPages(bool commit) { int dirty = 0; int result = _disk.WriteLogDisk(source()); _disk.DiscardCleanPages(from x in _snapshots.Values.Where((Snapshot x) => x.Mode == LockMode.Write).SelectMany((Snapshot x) => x.GetWritablePages(dirty: false, commit)) select x.Buffer); return result; IEnumerable source() { IEnumerable source = _snapshots.Values.Where((Snapshot x) => x.Mode == LockMode.Write).SelectMany((Snapshot x) => x.GetWritablePages(dirty: true, commit)); bool markLastAsConfirmed = commit && !_transPages.HeaderChanged; foreach (LastItem page in source.IsLast()) { page.Item.TransactionID = _transactionID; if (page.IsLast) { page.Item.IsConfirmed = markLastAsConfirmed; } if (_transPages.LastDeletedPageID == page.Item.PageID && commit) { Constants.ENSURE(_transPages.HeaderChanged, "must header be in lock"); Constants.ENSURE(page.Item.PageType == PageType.Empty, "must be marked as deleted page"); page.Item.NextPageID = _header.FreeEmptyPageList; _header.FreeEmptyPageList = _transPages.FirstDeletedPageID; } PageBuffer buffer = page.Item.UpdateBuffer(); yield return buffer; dirty++; _transPages.DirtyPages[page.Item.PageID] = new PagePosition(page.Item.PageID, buffer.Position); } if (commit && _transPages.HeaderChanged) { _header.TransactionID = _transactionID; _header.IsConfirmed = true; _transPages.OnCommit(_header); PageBuffer pageBuffer = _header.UpdateBuffer(); PageBuffer pageBuffer2 = _disk.NewPage(); Buffer.BlockCopy(pageBuffer.Array, pageBuffer.Offset, pageBuffer2.Array, pageBuffer2.Offset, pageBuffer2.Count); yield return pageBuffer2; } } } public void Commit() { Constants.ENSURE(_state == TransactionState.Active, "transaction must be active to commit (current state: {0})", _state); if (_mode == LockMode.Write || _transPages.HeaderChanged) { lock (_header) { if (PersistDirtyPages(commit: true) > 0) { _walIndex.ConfirmTransaction(_transactionID, _transPages.DirtyPages.Values); } } } foreach (Snapshot value in _snapshots.Values) { value.Dispose(); } _state = TransactionState.Committed; } public void Rollback() { Constants.ENSURE(_state == TransactionState.Active, "transaction must be active to rollback (current state: {0})", _state); if (_transPages.NewPages.Count > 0) { ReturnNewPages(); } foreach (Snapshot value in _snapshots.Values) { if (value.Mode == LockMode.Write) { _disk.DiscardDirtyPages(from x in value.GetWritablePages(dirty: true, includeCollectionPage: true) select x.Buffer); _disk.DiscardCleanPages(from x in value.GetWritablePages(dirty: false, includeCollectionPage: true) select x.Buffer); } value.Dispose(); } _state = TransactionState.Aborted; } private void ReturnNewPages() { uint transactionID = _walIndex.NextTransactionID(); Dictionary pagePositions; lock (_header) { pagePositions = new Dictionary(); PageBuffer savepoint = _header.Savepoint(); try { _disk.WriteLogDisk(source()); } catch { _header.Restore(savepoint); throw; } _walIndex.ConfirmTransaction(transactionID, pagePositions.Values); } IEnumerable source() { for (int i = 0; i < _transPages.NewPages.Count; i++) { uint pageID = _transPages.NewPages[i]; uint nextPageID = ((i < _transPages.NewPages.Count - 1) ? _transPages.NewPages[i + 1] : _header.FreeEmptyPageList); PageBuffer buffer = _disk.NewPage(); BasePage basePage = new BasePage(buffer, pageID, PageType.Empty) { NextPageID = nextPageID, TransactionID = transactionID }; yield return basePage.UpdateBuffer(); pagePositions[pageID] = new PagePosition(pageID, buffer.Position); } _header.TransactionID = transactionID; _header.FreeEmptyPageList = _transPages.NewPages[0]; _header.IsConfirmed = true; PageBuffer pageBuffer = _header.UpdateBuffer(); PageBuffer pageBuffer2 = _disk.NewPage(); Buffer.BlockCopy(pageBuffer.Array, pageBuffer.Offset, pageBuffer2.Array, pageBuffer2.Offset, pageBuffer2.Count); yield return pageBuffer2; } } public void Dispose() { Dispose(dispose: true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool dispose) { if (_state == TransactionState.Disposed) { return; } Constants.ENSURE(_state != TransactionState.Disposed, "transaction must be active before call Done"); if (_state == TransactionState.Active && _snapshots.Count > 0) { foreach (Snapshot item in _snapshots.Values.Where((Snapshot x) => x.Mode == LockMode.Write)) { _disk.DiscardDirtyPages(from x in item.GetWritablePages(dirty: true, includeCollectionPage: true) select x.Buffer); _disk.DiscardCleanPages(from x in item.GetWritablePages(dirty: false, includeCollectionPage: true) select x.Buffer); } foreach (Snapshot item2 in _snapshots.Values.Where((Snapshot x) => x.Mode == LockMode.Read)) { foreach (BasePage localPage in item2.LocalPages) { localPage.Buffer.Release(); } item2.CollectionPage?.Buffer.Release(); } } _reader.Dispose(); _state = TransactionState.Disposed; if (!dispose) { _monitor.ReleaseTransaction(this); } } } internal class WalIndexService { private readonly DiskService _disk; private readonly LockService _locker; private readonly Dictionary>> _index = new Dictionary>>(); private readonly ReaderWriterLockSlim _indexLock = new ReaderWriterLockSlim(); private readonly HashSet _confirmTransactions = new HashSet(); private int _currentReadVersion; private int _lastTransactionID; public int CurrentReadVersion { get { _indexLock.TryEnterReadLock(-1); try { return _currentReadVersion; } finally { _indexLock.ExitReadLock(); } } } public int LastTransactionID => _lastTransactionID; public WalIndexService(DiskService disk, LockService locker) { _disk = disk; _locker = locker; } public void Clear() { _indexLock.TryEnterWriteLock(-1); try { _confirmTransactions.Clear(); _index.Clear(); _lastTransactionID = 0; _currentReadVersion = 0; _disk.Cache.Clear(); _disk.SetLength(0L, FileOrigin.Log); } finally { _indexLock.ExitWriteLock(); } } public uint NextTransactionID() { return (uint)Interlocked.Increment(ref _lastTransactionID); } public long GetPageIndex(uint pageID, int version, out int walVersion) { if (version == 0) { walVersion = 0; return long.MaxValue; } _indexLock.TryEnterReadLock(-1); try { if (_index.TryGetValue(pageID, out var value)) { int num = value.Count; long result = long.MaxValue; walVersion = version; while (num > 0) { num--; KeyValuePair keyValuePair = value[num]; if (keyValuePair.Key <= version) { walVersion = keyValuePair.Key; result = keyValuePair.Value; break; } } return result; } walVersion = int.MaxValue; return long.MaxValue; } finally { _indexLock.ExitReadLock(); } } public void ConfirmTransaction(uint transactionID, ICollection pagePositions) { _indexLock.TryEnterWriteLock(-1); try { _currentReadVersion++; foreach (PagePosition pagePosition in pagePositions) { if (!_index.TryGetValue(pagePosition.PageID, out var value)) { value = new List>(); _index.Add(pagePosition.PageID, value); } value.Add(new KeyValuePair(_currentReadVersion, pagePosition.Position)); } _confirmTransactions.Add(transactionID); } finally { _indexLock.ExitWriteLock(); } } public void RestoreIndex(ref HeaderPage header) { Dictionary> dictionary = new Dictionary>(); long num = 0L; foreach (PageBuffer item2 in _disk.ReadFull(FileOrigin.Log)) { if (item2.IsBlank()) { num += 8192; continue; } uint pageID = item2.ReadUInt32(0); bool num2 = item2.ReadBool(18); uint num3 = item2.ReadUInt32(14); PagePosition item = new PagePosition(pageID, num); if (dictionary.TryGetValue(num3, out var value)) { value.Add(item); } else { dictionary[num3] = new List { item }; } if (num2) { ConfirmTransaction(num3, dictionary[num3]); if (item2.ReadByte(4) == 1) { PageBuffer buffer = header.Buffer; Buffer.BlockCopy(item2.Array, item2.Offset, buffer.Array, buffer.Offset, 8192); header = new HeaderPage(buffer); header.TransactionID = uint.MaxValue; header.IsConfirmed = false; } } _lastTransactionID = (int)num3; num += 8192; } } public int Checkpoint() { if (_disk.GetFileLength(FileOrigin.Log) == 0L || _confirmTransactions.Count == 0) { return 0; } bool flag = _locker.EnterExclusive(); try { return CheckpointInternal(); } finally { if (flag) { _locker.ExitExclusive(); } } } public int TryCheckpoint() { if (_disk.GetFileLength(FileOrigin.Log) == 0L || _confirmTransactions.Count == 0) { return 0; } if (!_locker.TryEnterExclusive(out var mustExit)) { return 0; } try { return CheckpointInternal(); } finally { if (mustExit) { _locker.ExitExclusive(); } } } private int CheckpointInternal() { int counter = 0; _disk.WriteDataDisk(source()); Clear(); return counter; IEnumerable source() { foreach (PageBuffer item2 in _disk.ReadFull(FileOrigin.Log)) { if (!item2.IsBlank()) { uint item = item2.ReadUInt32(14); if (_confirmTransactions.Contains(item)) { uint pageID = item2.ReadUInt32(0); item2.Write(uint.MaxValue, 14); item2.Write(value: false, 18); item2.Position = BasePage.GetPagePosition(pageID); counter++; yield return item2; } } } } } } internal class SortContainer : IDisposable { private readonly Collation _collation; private readonly int _size; private int _remaining; private int _count; private bool _isEOF; private int _readPosition; private BufferReader _reader; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public KeyValuePair Current; public bool IsEOF => _isEOF; public long Position { get; set; } = -1L; public int Count => _count; public SortContainer(Collation collation, int size) { _collation = collation; _size = size; } public void Insert(IEnumerable> items, int order, BufferSlice buffer) { IOrderedEnumerable> obj = ((order == 1) ? items.OrderBy((KeyValuePair x) => x.Key, _collation) : items.OrderByDescending((KeyValuePair x) => x.Key, _collation)); int num = 0; foreach (KeyValuePair item in obj) { buffer.WriteIndexKey(item.Key, num); int keyLength = IndexNode.GetKeyLength(item.Key, recalc: false); if (keyLength > 1023) { throw LiteException.InvalidIndexKey($"Sort key must be less than {1023} bytes."); } num += keyLength; buffer.Write(item.Value, num); num += 5; _remaining++; } _count = _remaining; } public void InitializeReader(Stream stream, BufferSlice buffer, bool utcDate) { if (stream != null) { _reader = new BufferReader(GetSourceFromStream(stream), utcDate); } else { _reader = new BufferReader(buffer, utcDate); } MoveNext(); } public bool MoveNext() { if (_remaining == 0) { _isEOF = true; return false; } BsonValue key = _reader.ReadIndexKey(); PageAddress value = _reader.ReadPageAddress(); Current = new KeyValuePair(key, value); _remaining--; return true; } private IEnumerable GetSourceFromStream(Stream stream) { byte[] bytes = _bufferPool.Rent(8192); BufferSlice buffer = new BufferSlice(bytes, 0, 8192); while (_readPosition < _size) { stream.Position = Position + _readPosition; stream.Read(bytes, 0, 8192); _readPosition += 8192; yield return buffer; } _bufferPool.Return(bytes, clearArray: true); } public void Dispose() { _reader?.Dispose(); } } internal class SortDisk : IDisposable { private readonly IStreamFactory _factory; private readonly StreamPool _pool; private readonly ConcurrentBag _freePositions = new ConcurrentBag(); private long _lastContainerPosition; private readonly int _containerSize; private readonly EnginePragmas _pragmas; public int ContainerSize => _containerSize; public SortDisk(IStreamFactory factory, int containerSize, EnginePragmas pragmas) { Constants.ENSURE(containerSize % 8192 == 0, "size must be PAGE_SIZE multiple"); _factory = factory; _containerSize = containerSize; _pragmas = pragmas; _lastContainerPosition = -containerSize; _pool = new StreamPool(_factory, appendOnly: false); } public Stream GetReader() { return _pool.Rent(); } public void Return(Stream stream) { _pool.Return(stream); } public void Return(long position) { _freePositions.Add(position); } public long GetContainerPosition() { if (_freePositions.TryTake(out var result)) { return result; } return Interlocked.Add(ref _lastContainerPosition, _containerSize); } public void Write(long position, BufferSlice buffer) { Stream value = _pool.Writer.Value; lock (value) { for (int i = 0; i < _containerSize / 8192; i++) { value.Position = position + i * 8192; value.Write(buffer.Array, buffer.Offset + i * 8192, 8192); } } } public void Dispose() { _pool.Dispose(); _factory.Delete(); } } internal class SortService : IDisposable { private readonly SortDisk _disk; private readonly List _containers = new List(); private readonly int _containerSize; private readonly Done _done = new Done { Running = true }; private readonly int _order; private readonly EnginePragmas _pragmas; private readonly BufferSlice _buffer; private readonly Lazy _reader; private static readonly ArrayPool _bufferPool = ArrayPool.Shared; public int Count => _done.Count; public IReadOnlyCollection Containers => _containers; public SortService(SortDisk disk, int order, EnginePragmas pragmas) { _disk = disk; _order = order; _pragmas = pragmas; _containerSize = disk.ContainerSize; _reader = new Lazy(() => _disk.GetReader()); byte[] array = new byte[disk.ContainerSize]; _buffer = new BufferSlice(array, 0, _containerSize); } public void Dispose() { foreach (SortContainer container in _containers) { container.Dispose(); if (container.Position >= 0) { _disk.Return(container.Position); } } if (_reader.IsValueCreated) { _disk.Return(_reader.Value); } } public void Insert(IEnumerable> items) { foreach (IEnumerable> item in SliptValues(items, _done)) { SortContainer sortContainer = new SortContainer(_pragmas.Collation, _containerSize); sortContainer.Insert(item, _order, _buffer); _containers.Add(sortContainer); if (!_done.Running && _containers.Count == 1) { sortContainer.InitializeReader(null, _buffer, _pragmas.UtcDate); continue; } sortContainer.Position = _disk.GetContainerPosition(); _disk.Write(sortContainer.Position, _buffer); sortContainer.InitializeReader(_reader.Value, null, _pragmas.UtcDate); } } public IEnumerable> Sort() { if (_containers.Count == 0) { yield break; } SortContainer current = _containers[0]; if (_containers.Count == 1) { do { yield return current.Current; } while (current.MoveNext()); current.Dispose(); yield break; } int diffOrder = _order * -1; while (_containers.Any((SortContainer x) => !x.IsEOF)) { foreach (SortContainer item in _containers.Where((SortContainer x) => !x.IsEOF)) { if (item.Current.Key.CompareTo(current.Current.Key, _pragmas.Collation) == diffOrder) { current = item; } } yield return current.Current; BsonValue lastKey = current.Current.Key; if (!current.MoveNext()) { current = _containers.FirstOrDefault((SortContainer x) => !x.IsEOF); } while (current?.Current.Key == lastKey) { yield return current.Current; if (!current.MoveNext()) { current = _containers.FirstOrDefault((SortContainer x) => !x.IsEOF); } } } } private IEnumerable>> SliptValues(IEnumerable> source, Done done) { using IEnumerator> enumerator = source.GetEnumerator(); if (enumerator.MoveNext()) { done.Count = 1; while (done.Running) { yield return YieldValues(enumerator, done); } } } private IEnumerable> YieldValues(IEnumerator> source, Done done) { int size = IndexNode.GetKeyLength(source.Current.Key, recalc: false) + 5; yield return source.Current; while (source.MoveNext()) { int num = IndexNode.GetKeyLength(source.Current.Key, recalc: false) + 5; done.Count++; if (size + num > _containerSize) { yield break; } size += num; yield return source.Current; } done.Running = false; } } internal class CollectionIndex { public uint FreeIndexPageList; public byte Slot { get; } public byte IndexType { get; } public string Name { get; } public string Expression { get; } public BsonExpression BsonExpr { get; } public bool Unique { get; } public PageAddress Head { get; set; } public PageAddress Tail { get; set; } public byte Reserved { get; set; } = 1; public bool IsEmpty => string.IsNullOrEmpty(Name); public CollectionIndex(byte slot, byte indexType, string name, string expr, bool unique) { Slot = slot; IndexType = indexType; Name = name; Expression = expr; Unique = unique; FreeIndexPageList = uint.MaxValue; BsonExpr = BsonExpression.Create(expr); } public CollectionIndex(BufferReader reader) { Slot = reader.ReadByte(); IndexType = reader.ReadByte(); Name = reader.ReadCString(); Expression = reader.ReadCString(); Unique = reader.ReadBoolean(); Head = reader.ReadPageAddress(); Tail = reader.ReadPageAddress(); Reserved = reader.ReadByte(); FreeIndexPageList = reader.ReadUInt32(); BsonExpr = BsonExpression.Create(Expression); } public void UpdateBuffer(BufferWriter writer) { writer.Write(Slot); writer.Write(IndexType); writer.WriteCString(Name); writer.WriteCString(Expression); writer.Write(Unique); writer.Write(Head); writer.Write(Tail); writer.Write(Reserved); writer.Write(FreeIndexPageList); } public static int GetLength(CollectionIndex index) { return GetLength(index.Name, index.Expression); } public static int GetLength(string name, string expr) { return 2 + StringEncoding.UTF8.GetByteCount(name) + 1 + StringEncoding.UTF8.GetByteCount(expr) + 1 + 1 + 5 + 5 + 1 + 4; } } internal class CursorInfo { public string Collection { get; } public Query Query { get; set; } public int Fetched { get; set; } public Stopwatch Elapsed { get; } = new Stopwatch(); public CursorInfo(string collection, Query query) { Collection = collection; Query = query; } } internal class DataBlock { public const int DATA_BLOCK_FIXED_SIZE = 6; public const int P_EXTEND = 0; public const int P_NEXT_BLOCK = 1; public const int P_BUFFER = 6; private readonly DataPage _page; private readonly BufferSlice _segment; public PageAddress Position { get; } public bool Extend { get; } public PageAddress NextBlock { get; private set; } public BufferSlice Buffer { get; } public DataBlock(DataPage page, byte index, BufferSlice segment) { _page = page; _segment = segment; Position = new PageAddress(page.PageID, index); Extend = segment.ReadBool(0); NextBlock = segment.ReadPageAddress(1); Buffer = segment.Slice(6, segment.Count - 6); } public DataBlock(DataPage page, byte index, BufferSlice segment, bool extend, PageAddress nextBlock) { _page = page; _segment = segment; Position = new PageAddress(page.PageID, index); NextBlock = nextBlock; Extend = extend; segment.Write(extend, 0); segment.Write(nextBlock, 1); Buffer = segment.Slice(6, segment.Count - 6); page.IsDirty = true; } public void SetNextBlock(PageAddress nextBlock) { NextBlock = nextBlock; _segment.Write(nextBlock, 1); _page.IsDirty = true; } public override string ToString() { return $"Pos: [{Position}] - Ext: [{Extend}] - Next: [{NextBlock}] - Buffer: [{Buffer}]"; } } internal class Done { public bool Running; public int Count; } public enum FileOrigin : byte { None, Data, Log } internal class IndexNode { private const int INDEX_NODE_FIXED_SIZE = 12; private const int P_SLOT = 0; private const int P_LEVELS = 1; private const int P_DATA_BLOCK = 2; private const int P_NEXT_NODE = 7; private const int P_PREV_NEXT = 12; private readonly IndexPage _page; private readonly BufferSlice _segment; private static readonly byte[] arrayByteEmpty = new byte[0]; private int P_KEY => 12 + Levels * 5 * 2; public PageAddress Position { get; } public byte Slot { get; } public byte Levels { get; } public BsonValue Key { get; } public PageAddress DataBlock { get; } public PageAddress NextNode { get; private set; } public PageAddress[] Prev { get; private set; } public PageAddress[] Next { get; private set; } public IndexPage Page => _page; public static int GetNodeLength(byte level, BsonValue key, out int keyLength) { keyLength = GetKeyLength(key, recalc: true); return 12 + level * 2 * 5 + keyLength; } public static int GetKeyLength(BsonValue key, bool recalc) { return 1 + ((key.IsString || key.IsBinary) ? 1 : 0) + key.GetBytesCount(recalc); } public IndexNode(IndexPage page, byte index, BufferSlice segment) { _page = page; _segment = segment; Position = new PageAddress(page.PageID, index); Slot = segment.ReadByte(0); Levels = segment.ReadByte(1); DataBlock = segment.ReadPageAddress(2); NextNode = segment.ReadPageAddress(7); Next = new PageAddress[Levels]; Prev = new PageAddress[Levels]; for (int i = 0; i < Levels; i++) { Prev[i] = segment.ReadPageAddress(12 + i * 5 * 2); Next[i] = segment.ReadPageAddress(12 + i * 5 * 2 + 5); } Key = segment.ReadIndexKey(P_KEY); } public IndexNode(IndexPage page, byte index, BufferSlice segment, byte slot, byte levels, BsonValue key, PageAddress dataBlock) { _page = page; _segment = segment; Position = new PageAddress(page.PageID, index); Slot = slot; Levels = levels; DataBlock = dataBlock; NextNode = PageAddress.Empty; Next = new PageAddress[levels]; Prev = new PageAddress[levels]; Key = key; segment.Write(slot, 0); segment.Write(levels, 1); segment.Write(dataBlock, 2); segment.Write(NextNode, 7); for (int i = 0; i < levels; i++) { SetPrev((byte)i, PageAddress.Empty); SetNext((byte)i, PageAddress.Empty); } segment.WriteIndexKey(key, P_KEY); page.IsDirty = true; } public IndexNode(BsonDocument doc) { _page = null; _segment = new BufferSlice(arrayByteEmpty, 0, 0); Position = new PageAddress(0u, 0); Slot = 0; Levels = 0; DataBlock = PageAddress.Empty; NextNode = PageAddress.Empty; Next = new PageAddress[0]; Prev = new PageAddress[0]; Key = doc; } public void SetNextNode(PageAddress value) { NextNode = value; _segment.Write(value, 7); _page.IsDirty = true; } public void SetPrev(byte level, PageAddress value) { Constants.ENSURE(level <= Levels, "out of index in level"); Prev[level] = value; _segment.Write(value, 12 + level * 5 * 2); _page.IsDirty = true; } public void SetNext(byte level, PageAddress value) { Constants.ENSURE(level <= Levels, "out of index in level"); Next[level] = value; _segment.Write(value, 12 + level * 5 * 2 + 5); _page.IsDirty = true; } public PageAddress GetNextPrev(byte level, int order) { if (order != 1) { return Prev[level]; } return Next[level]; } public override string ToString() { return $"Pos: [{Position}] - Key: {Key}"; } } internal enum LockMode { Read, Write } [DebuggerStepThrough] internal struct PageAddress { public const int SIZE = 5; public static PageAddress Empty = new PageAddress(uint.MaxValue, byte.MaxValue); public readonly uint PageID; public readonly byte Index; public bool IsEmpty { get { if (PageID == uint.MaxValue) { return Index == byte.MaxValue; } return false; } } public override bool Equals(object obj) { PageAddress pageAddress = (PageAddress)obj; if (PageID == pageAddress.PageID) { return Index == pageAddress.Index; } return false; } public static bool operator ==(PageAddress lhs, PageAddress rhs) { if (lhs.PageID == rhs.PageID) { return lhs.Index == rhs.Index; } return false; } public static bool operator !=(PageAddress lhs, PageAddress rhs) { return !(lhs == rhs); } public override int GetHashCode() { return (int)((17 * 23 + PageID) * 23 + Index); } public PageAddress(uint pageID, byte index) { PageID = pageID; Index = index; } public override string ToString() { if (!IsEmpty) { uint pageID = PageID; string text = pageID.ToString().PadLeft(4, '0'); byte index = Index; return text + ":" + index.ToString().PadLeft(2, '0'); } return "(empty)"; } public BsonValue ToBsonValue() { if (IsEmpty) { return BsonValue.Null; } return new BsonDocument { ["pageID"] = (int)PageID, ["index"] = (int)Index }; } } internal class PageBuffer : BufferSlice { public readonly int UniqueID; public long Position; public FileOrigin Origin; public int ShareCounter; public long Timestamp; public PageBuffer(byte[] buffer, int offset, int uniqueID) : base(buffer, offset, 8192) { UniqueID = uniqueID; Position = long.MaxValue; Origin = FileOrigin.None; ShareCounter = 0; Timestamp = 0L; } public void Release() { Constants.ENSURE(ShareCounter > 0, "share counter must be > 0 in Release()"); Interlocked.Decrement(ref ShareCounter); } public override string ToString() { string text = ((Position == long.MaxValue) ? "" : Position.ToString()); string text2 = ((ShareCounter == Constants.BUFFER_WRITABLE) ? "" : ShareCounter.ToString()); uint num = this.ReadUInt32(0); byte b = base[4]; return string.Format("ID: {0} - Position: {1}/{2} - Shared: {3} - ({4}) :: Content: [{5}/{6}]", UniqueID, text, Origin, text2, base.ToString(), num.ToString("0:0000"), (PageType)b); } public unsafe bool IsBlank() { fixed (byte* ptr = base.Array) { ulong* ptr2 = (ulong*)(ptr + base.Offset); if (*ptr2 == 0L) { return ptr2[1] == 0; } return false; } } } internal struct PagePosition { public uint PageID; public long Position; public static PagePosition Empty => new PagePosition(uint.MaxValue, long.MaxValue); public bool IsEmpty { get { if (PageID == uint.MaxValue) { return Position == long.MaxValue; } return false; } } public override bool Equals(object obj) { PagePosition pagePosition = (PagePosition)obj; if (PageID == pagePosition.PageID) { return Position == pagePosition.PageID; } return false; } public override int GetHashCode() { return (int)((17 * 23 + PageID) * 23) + (int)Position; } public PagePosition(uint pageID, long position) { PageID = pageID; Position = position; } public override string ToString() { if (!IsEmpty) { return ((PageID == uint.MaxValue) ? "----" : PageID.ToString()) + ":" + Position; } return "----:----"; } } internal class Pragma { public string Name { get; set; } public Func Get { get; set; } public Action Set { get; set; } public Action Read { get; set; } public Action Validate { get; set; } public Action Write { get; set; } } public class RebuildOptions { private string _buildId = Guid.NewGuid().ToString("d").ToLower() .Substring(6); public string Password { get; set; } public Collation Collation { get; set; } public bool IncludeErrorReport { get; set; } = true; internal IList Errors { get; } = new List(); public IEnumerable GetErrorReport() { return Errors.Select((FileReaderError x) => new BsonDocument { ["buildId"] = _buildId, ["created"] = x.Created, ["pageID"] = (int)x.PageID.Value, ["positionID"] = x.Position, ["origin"] = x.Origin.ToString(), ["pageType"] = x.PageType.ToString(), ["message"] = x.Message, ["exception"] = new BsonDocument { ["code"] = ((x.Exception is LiteException ex) ? ex.ErrorCode : (-1)), ["hresult"] = x.Exception.HResult, ["type"] = x.Exception.GetType().FullName, ["inner"] = x.Exception.InnerException?.Message, ["stacktrace"] = x.Exception.StackTrace } }); } } internal class TransactionPages { public int TransactionSize { get; set; } public Dictionary DirtyPages { get; } = new Dictionary(); public List NewPages { get; } = new List(); public uint FirstDeletedPageID { get; set; } = uint.MaxValue; public uint LastDeletedPageID { get; set; } = uint.MaxValue; public int DeletedPages { get; set; } public bool HeaderChanged { get { if (NewPages.Count <= 0 && DeletedPages <= 0) { return this.Commit != null; } return true; } } public event Action Commit; public void OnCommit(HeaderPage header) { this.Commit?.Invoke(header); } } internal enum TransactionState { Active, Committed, Aborted, Disposed } internal class SysDump : SystemCollection { private readonly HeaderPage _header; private readonly TransactionMonitor _monitor; public SysDump(HeaderPage header, TransactionMonitor monitor) : base("$dump") { _header = header; _monitor = monitor; } public override IEnumerable Input(BsonValue options) { BsonValue option = SystemCollection.GetOption(options, "pageID"); return DumpPages((option == null) ? null : new uint?((uint)option.AsInt32)); } private IEnumerable DumpPages(uint? pageID) { Dictionary collections = _header.GetCollections().ToDictionary((KeyValuePair x) => x.Value, (KeyValuePair x) => x.Key); TransactionService transaction = _monitor.GetThreadTransaction(); Snapshot snapshot = transaction.CreateSnapshot(LockMode.Read, "$", addIfNotExists: false); uint num = (pageID.HasValue ? pageID.Value : 0u); uint end = (pageID.HasValue ? pageID.Value : _header.LastPageID); for (uint i = num; i <= Math.Min(end, _header.LastPageID); i++) { FileOrigin origin; long position; int walVersion; BasePage page = snapshot.GetPage(i, out origin, out position, out walVersion); BsonDocument bsonDocument = new BsonDocument { ["pageID"] = (int)page.PageID, ["pageType"] = page.PageType.ToString(), ["_position"] = position, ["_origin"] = origin.ToString(), ["_version"] = walVersion, ["prevPageID"] = (int)page.PrevPageID, ["nextPageID"] = (int)page.NextPageID, ["slot"] = (int)page.PageListSlot, ["collection"] = collections.GetOrDefault(page.ColID, "-"), ["itemsCount"] = (int)page.ItemsCount, ["freeBytes"] = page.FreeBytes, ["usedBytes"] = (int)page.UsedBytes, ["fragmentedBytes"] = (int)page.FragmentedBytes, ["nextFreePosition"] = (int)page.NextFreePosition, ["highestIndex"] = (int)page.HighestIndex }; if (page.PageType == PageType.Collection) { CollectionPage collectionPage = new CollectionPage(page.Buffer); bsonDocument["dataPageList"] = new BsonArray(collectionPage.FreeDataPageList.Select((uint x) => new BsonValue((int)x))); bsonDocument["indexes"] = new BsonArray(from x in collectionPage.GetCollectionIndexes() select new BsonDocument { ["slot"] = (int)x.Slot, ["empty"] = x.IsEmpty, ["indexType"] = (int)x.IndexType, ["name"] = x.Name, ["expression"] = x.Expression, ["unique"] = x.Unique, ["head"] = x.Head.ToBsonValue(), ["tail"] = x.Tail.ToBsonValue(), ["freeIndexPageList"] = (int)x.FreeIndexPageList }); } if (pageID.HasValue) { bsonDocument["buffer"] = page.Buffer.ToArray(); } yield return bsonDocument; transaction.Safepoint(); } } } internal class SysFile : SystemCollection { private readonly Dictionary _formats = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["json"] = new SysFileJson(), ["csv"] = new SysFileCsv() }; public SysFile() : base("$file") { } public override IEnumerable Input(BsonValue options) { string format = GetFormat(options); if (_formats.TryGetValue(format, out var value)) { return value.Input(options); } throw new LiteException(0, "Unknow file format in $file: `" + format + "`"); } public override int Output(IEnumerable source, BsonValue options) { string format = GetFormat(options); if (_formats.TryGetValue(format, out var value)) { return value.Output(source, options); } throw new LiteException(0, "Unknow file format in $file: `" + format + "`"); } private string GetFormat(BsonValue options) { string path = SystemCollection.GetOption(options, "filename")?.AsString ?? throw new LiteException(0, "Collection $file requires string as 'filename' or a document field 'filename'"); string asString = SystemCollection.GetOption(options, "format", Path.GetExtension(path)).AsString; if (!asString.StartsWith(".")) { return asString; } return asString.Substring(1); } } internal class SysFileCsv : SystemCollection { private static readonly IFormatProvider _numberFormat = CultureInfo.InvariantCulture.NumberFormat; public SysFileCsv() : base("$file_csv") { } public override IEnumerable Input(BsonValue options) { string path = SystemCollection.GetOption(options, "filename")?.AsString ?? throw new LiteException(0, "Collection $" + base.Name + " requires string as 'filename' or a document field 'filename'"); string asString = SystemCollection.GetOption(options, "encoding", "utf-8").AsString; char delimiter = SystemCollection.GetOption(options, "delimiter", ",").AsString[0]; List header = new List(); if (options.IsDocument && options["header"].IsArray) { header.AddRange(options["header"].AsArray.Select((BsonValue x) => x.AsString)); } using FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); using StreamReader reader = new StreamReader(fs, Encoding.GetEncoding(asString)); if (header.Count == 0) { bool newLine; do { string text = ReadString(reader, delimiter, out newLine); if (text == null) { break; } header.Add(text); } while (!newLine); } int num = 0; BsonDocument bsonDocument = new BsonDocument(); while (true) { bool newLine2; string text2 = ReadString(reader, delimiter, out newLine2); if (text2 != null) { if (num < header.Count) { string name = header[num++]; bsonDocument[name] = text2; } if (newLine2) { yield return bsonDocument; bsonDocument = new BsonDocument(); num = 0; } continue; } break; } } public override int Output(IEnumerable source, BsonValue options) { string path = SystemCollection.GetOption(options, "filename")?.AsString ?? throw new LiteException(0, "Collection $file_json requires string as 'filename' or a document field 'filename'"); bool asBoolean = SystemCollection.GetOption(options, "overwritten", false).AsBoolean; string asString = SystemCollection.GetOption(options, "encoding", "utf-8").AsString; char value = SystemCollection.GetOption(options, "delimiter", ",").AsString[0]; bool asBoolean2 = SystemCollection.GetOption(options, "header", true).AsBoolean; int num = 0; IList list = null; FileStream fileStream = null; StreamWriter streamWriter = null; try { foreach (BsonDocument item in source) { if (num++ == 0) { fileStream = new FileStream(path, (!asBoolean) ? FileMode.CreateNew : FileMode.OpenOrCreate); streamWriter = new StreamWriter(fileStream, Encoding.GetEncoding(asString)); list = item.Keys.ToList(); if (asBoolean2) { int num2 = 0; foreach (KeyValuePair item2 in item) { if (num2++ > 0) { streamWriter.Write(value); } streamWriter.Write(item2.Key); } streamWriter.WriteLine(); } } else { streamWriter.WriteLine(); } int num3 = 0; foreach (string item3 in list) { BsonValue value2 = item[item3]; if (num3++ > 0) { streamWriter.Write(value); } WriteValue(value2, streamWriter); } } if (num > 0) { streamWriter.Flush(); } } finally { streamWriter?.Dispose(); fileStream?.Dispose(); } return num; } private void WriteValue(BsonValue value, StreamWriter writer) { switch (value.Type) { case BsonType.MinValue: case BsonType.Null: case BsonType.Document: case BsonType.Array: case BsonType.MaxValue: writer.Write("null"); break; case BsonType.Int64: writer.Write(value.AsInt64.ToString(CultureInfo.InvariantCulture.NumberFormat)); break; case BsonType.Decimal: writer.Write(value.AsDecimal.ToString(CultureInfo.InvariantCulture.NumberFormat)); break; case BsonType.Binary: writer.Write("\"" + Convert.ToBase64String(value.AsBinary) + "\""); break; case BsonType.ObjectId: writer.Write("\"" + value.AsObjectId.ToString() + "\""); break; case BsonType.Guid: writer.Write("\"" + value.AsGuid.ToString() + "\""); break; case BsonType.DateTime: writer.Write("\"" + value.AsDateTime.ToUniversalTime().ToString("o") + "\""); break; default: writer.Write(value.ToString()); break; } } private string ReadString(TextReader reader, char delimiter, out bool newLine) { StringBuilder stringBuilder = new StringBuilder(); int num = reader.Read(); while (num == 10 || num == 13) { num = reader.Read(); } if (num == -1) { newLine = true; return null; } if (num == 34) { while (num != -1) { num = reader.Read(); if (num == 34) { int num2 = reader.Read(); if (num2 != 34) { num = num2; break; } stringBuilder.Append('"'); } else { stringBuilder.Append((char)num); } } } else { while (num != 10 && num != 13 && num != delimiter && num != -1) { stringBuilder.Append((char)num); num = reader.Read(); } } newLine = num == 10 || num == 13; return stringBuilder.ToString(); } } internal class SysFileJson : SystemCollection { public SysFileJson() : base("$file_json") { } public override IEnumerable Input(BsonValue options) { string path = SystemCollection.GetOption(options, "filename")?.AsString ?? throw new LiteException(0, "Collection $" + base.Name + " requires string as 'filename' or a document field 'filename'"); string asString = SystemCollection.GetOption(options, "encoding", "utf-8").AsString; using FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); using StreamReader reader = new StreamReader(fs, Encoding.GetEncoding(asString)); IEnumerable enumerable = from x in new JsonReader(reader).DeserializeArray() select x.AsDocument; foreach (BsonDocument item in enumerable) { yield return item; } } public override int Output(IEnumerable source, BsonValue options) { string path = SystemCollection.GetOption(options, "filename")?.AsString ?? throw new LiteException(0, "Collection $file_json requires string as filename or a document field 'filename'"); bool asBoolean = SystemCollection.GetOption(options, "pretty", false).AsBoolean; int asInt = SystemCollection.GetOption(options, "indent", 4).AsInt32; string asString = SystemCollection.GetOption(options, "encoding", "utf-8").AsString; bool asBoolean2 = SystemCollection.GetOption(options, "overwritten", false).AsBoolean; int num = 0; FileStream fileStream = null; StreamWriter streamWriter = null; JsonWriter jsonWriter = null; try { foreach (BsonDocument item in source) { if (num++ == 0) { fileStream = new FileStream(path, (!asBoolean2) ? FileMode.CreateNew : FileMode.OpenOrCreate); streamWriter = new StreamWriter(fileStream, Encoding.GetEncoding(asString)); jsonWriter = new JsonWriter(streamWriter) { Pretty = asBoolean, Indent = asInt }; streamWriter.WriteLine("["); } else { streamWriter.WriteLine(","); } jsonWriter.Serialize(item); } if (num > 0) { streamWriter.WriteLine(); streamWriter.Write("]"); streamWriter.Flush(); } } finally { streamWriter?.Dispose(); fileStream?.Dispose(); } return num; } } internal class SysPageList : SystemCollection { private readonly HeaderPage _header; private readonly TransactionMonitor _monitor; private Dictionary _collections; public SysPageList(HeaderPage header, TransactionMonitor monitor) : base("$page_list") { _header = header; _monitor = monitor; } public override IEnumerable Input(BsonValue options) { BsonValue option = SystemCollection.GetOption(options, "pageID"); TransactionService threadTransaction = _monitor.GetThreadTransaction(); Snapshot snapshot = threadTransaction.CreateSnapshot(LockMode.Read, "$", addIfNotExists: false); _collections = _header.GetCollections().ToDictionary((KeyValuePair x) => x.Value, (KeyValuePair x) => x.Key); IEnumerable enumerable = ((option != null) ? GetList((uint)option.AsInt32, null, threadTransaction, snapshot) : GetAllList(threadTransaction, snapshot)); foreach (BsonDocument item in enumerable) { yield return item; } } private IEnumerable GetAllList(TransactionService transaction, Snapshot snapshot) { foreach (BsonDocument item in GetList(_header.FreeEmptyPageList, null, transaction, snapshot)) { yield return item; } foreach (KeyValuePair collection in _collections) { Snapshot snap = transaction.CreateSnapshot(LockMode.Read, collection.Value, addIfNotExists: false); for (int slot2 = 0; slot2 < 5; slot2++) { IEnumerable list = GetList(snap.CollectionPage.FreeDataPageList[slot2], null, transaction, snap); foreach (BsonDocument item2 in list) { yield return item2; } } CollectionIndex[] array = snap.CollectionPage.GetCollectionIndexes().ToArray(); CollectionIndex[] array2 = array; foreach (CollectionIndex collectionIndex in array2) { IEnumerable list2 = GetList(collectionIndex.FreeIndexPageList, collectionIndex.Name, transaction, snap); foreach (BsonDocument item3 in list2) { yield return item3; } } } } private IEnumerable GetList(uint pageID, string indexName, TransactionService transaction, Snapshot snapshot) { if (pageID == uint.MaxValue) { yield break; } BasePage page = snapshot.GetPage(pageID); while (page != null) { _collections.TryGetValue(page.ColID, out var value); yield return new BsonDocument { ["pageID"] = (int)page.PageID, ["pageType"] = page.PageType.ToString(), ["slot"] = (int)page.PageListSlot, ["collection"] = value, ["index"] = indexName, ["freeBytes"] = page.FreeBytes, ["itemsCount"] = (int)page.ItemsCount }; if (page.NextPageID != uint.MaxValue) { transaction.Safepoint(); page = snapshot.GetPage(page.NextPageID); continue; } break; } } } internal class SysQuery : SystemCollection { private readonly ILiteEngine _engine; public SysQuery(ILiteEngine engine) : base("$query") { _engine = engine; } public override IEnumerable Input(BsonValue options) { string source = options?.AsString ?? throw new LiteException(0, "Collection $query(sql) requires `sql` string parameter"); SqlParser sqlParser = new SqlParser(_engine, new Tokenizer(source), null); using IBsonDataReader reader = sqlParser.Execute(); while (reader.Read()) { BsonValue current = reader.Current; yield return current.IsDocument ? current.AsDocument : new BsonDocument { ["expr"] = current }; } } } internal class SystemCollection { private readonly string _name; private readonly Func> _input; public string Name => _name; public SystemCollection(string name) { if (!name.StartsWith("$")) { throw new ArgumentException("System collection name must starts with $"); } _name = name; } public SystemCollection(string name, Func> input) : this(name) { _input = input; } public virtual IEnumerable Input(BsonValue options) { return _input(); } public virtual int Output(IEnumerable source, BsonValue options) { throw new LiteException(0, _name + " do not support as output collection"); } protected static BsonValue GetOption(BsonValue options, string key) { return GetOption(options, key, null); } protected static BsonValue GetOption(BsonValue options, string key, BsonValue defaultValue) { if (options != null && options.IsDocument) { if (options.AsDocument.TryGetValue(key, out var value)) { if (defaultValue == null || value.Type == defaultValue.Type) { return value; } throw new LiteException(0, $"Parameter `{key}` expect {defaultValue.Type} value type"); } return defaultValue; } if (!(defaultValue == null)) { return defaultValue; } return options; } } }