commit 94be6c46bbb811dd615b9f575fb2437d38597cc3
Author: Leah (ctucx) <leah@ctu.cx>
Date: Tue, 17 May 2022 15:38:48 +0200
Author: Leah (ctucx) <leah@ctu.cx>
Date: Tue, 17 May 2022 15:38:48 +0200
initial commit!
11 files changed, 2030 insertions(+), 0 deletions(-)
A
|
197
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
273
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
128
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
663
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
182
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
81
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
|
264
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
diff --git a/.gitignore b/.gitignore @@ -0,0 +1,4 @@ + +website +mastofetch +hornposts.db+ \ No newline at end of file
diff --git a/libs/moustachu.nim b/libs/moustachu.nim @@ -0,0 +1,197 @@ +## This is basicly the code from fenekku's moustachu project with a few changes +## you can find it here: https://github.com/fenekku/moustachu + +## A mustache templating engine written in Nim. +import strutils, sequtils, json +import moustachu_context, moustachu_tokenizer + +export moustachu_context + +proc lookupContext (contextStack: seq[Context], tagkey: string): Context = + ## Return the Context associated with `tagkey` where `tagkey` + ## can be a dotted tag e.g. a.b.c + ## If the Context at `tagkey` does not exist, return nil. + var currCtx = contextStack[contextStack.high] + if tagkey == ".": return currCtx + let subtagkeys = tagkey.split(".") + for i in countDown(contextStack.high, contextStack.low): + currCtx = contextStack[i] + + for subtagkey in subtagkeys: + currCtx = currCtx[subtagkey] + if currCtx == nil: + break + + if currCtx != nil: + return currCtx + + return currCtx + +proc lookupString (contextStack: seq[Context], tagkey: string): string = + ## Return the string associated with `tagkey` in Context `c`. + ## If the Context at `tagkey` does not exist, return the empty string. + result = lookupContext(contextStack, tagkey).toString() + +proc ignore (tag: string, tokens: seq[Token], index: int): int = + #ignore + var i = index + 1 + var nexttoken = tokens[i] + var openedsections = 1 + let lentokens = len(tokens) + + while i < lentokens and openedsections > 0: + if nexttoken.value == tag: + if nexttoken.tokentype in [TokenType.section, TokenType.invertedsection]: + openedsections += 1 + elif nexttoken.tokentype == TokenType.ender: + openedsections -= 1 + else: discard + else: discard + + i += 1 + if i < lentokens: + nexttoken = tokens[i] + + return i + +proc parallelReplace (str: string, substitutions: openArray[tuple[pattern: string, by: string]]): string = + ## Returns a modified copy of `str` with the `substitutions` applied + result = str + for sub in substitutions: + result = result.replace(sub[0], sub[1]) + +proc render* (tmplate: string, contextStack: seq[Context]): string = + ## Take a mustache template `tmplate`, a seq of evaluation Context's + ## and return the rendered string. This is the main procedure. + let + htmlReplaceBy = [("&", "&"), + ("<", "<"), + (">", ">"), + ("\\", "\"), + ("\"", """)] + + var renderings : seq[string] = @[] + + #Object + var sections : seq[string] = @[] + var contextStack = contextStack + + #Array + var loopStartPositions : seq[int] = @[] + var loopCounters : seq[int] = @[] + + #Indentation + var indentation = "" + + let tokens = toSeq(moustachu_tokenizer.tokenize(tmplate)) + let lentokens = len(tokens) + + var index = 0 + + while index < lentokens: + let token = tokens[index] + + case token.tokentype + of TokenType.comment: + discard + + of TokenType.escapedvariable: + var viewvalue = contextStack.lookupString(token.value) +# viewvalue = viewvalue.parallelReplace(htmlReplaceBy) + renderings.add(viewvalue) + + of TokenType.unescapedvariable: + var viewvalue = contextStack.lookupString(token.value) + renderings.add(viewvalue) + + of TokenType.section: + let ctx = contextStack.lookupContext(token.value) + if ctx == nil: + index = ignore(token.value, tokens, index) + continue + elif ctx.kind == CObject: + # enter a new section + if ctx.len == 0: + index = ignore(token.value, tokens, index) + continue + else: + contextStack.add(ctx) + sections.add(token.value) + elif ctx.kind == CArray: + # update the array loop stacks + if ctx.len == 0: + index = ignore(token.value, tokens, index) + continue + else: + #do looping + index += 1 + loopStartPositions.add(index) + loopCounters.add(ctx.len) + sections.add(token.value) + contextStack.add(ctx[ctx.len - loopCounters[^1]]) + continue + elif ctx.kind == CValue: + if not ctx: + index = ignore(token.value, tokens, index) + continue + else: discard #we will render the text inside the section + + of TokenType.invertedsection: + let ctx = contextStack.lookupContext(token.value) + if ctx != nil: + if ctx.kind == CObject: + index = ignore(token.value, tokens, index) + continue + elif ctx.kind == CArray: + if ctx.len != 0: + index = ignore(token.value, tokens, index) + continue + elif ctx.kind == CValue: + if ctx: + index = ignore(token.value, tokens, index) + continue + else: discard #we will render the text inside the section + + of TokenType.ender: + var ctx = contextStack.lookupContext(token.value) + if ctx != nil: + if ctx.kind == CObject: + discard contextStack.pop() + discard sections.pop() + elif ctx.kind == CArray: + if ctx.len > 0: + loopCounters[^1] -= 1 + discard contextStack.pop() + if loopCounters[^1] == 0: + discard loopCounters.pop() + discard loopStartPositions.pop() + discard sections.pop() + else: + index = loopStartPositions[^1] + contextStack.add(ctx[ctx.len - loopCounters[^1]]) + continue + + of TokenType.indenter: + if token.value != "": + indentation = token.value + renderings.add(indentation) + + else: + renderings.add(token.value) + + index += 1 + + result = join(renderings, "") + +proc render* (tmplate: string, c: Context): string = + ## Take a mustache template `tmplate`, an evaluation Context `c` + ## and return the rendered string. + var contextStack = @[c] + result = tmplate.render(contextStack) + +proc render* (tmplate: string, jsonContext: JsonNode): string {.exportc.} = + ## Take a mustache template `tmplate`, an evaluation context as a JsonNode + ## and return the rendered string. + let nc = newContext(jsonContext) + var contextStack = @[nc] + result = tmplate.render(contextStack)
diff --git a/libs/moustachu_context.nim b/libs/moustachu_context.nim @@ -0,0 +1,273 @@ +import json, sequtils, strutils, tables + +type + ContextKind* = enum ## possible Context types + CArray, + CObject, + CValue + + ## Context used to render a mustache template + Context* = ref ContextObj + ContextObj = object + case kind*: ContextKind + of CValue: + val: JsonNode + of CArray: + elems: seq[Context] + of CObject: + fields: Table[string, Context] + +## Builders + +proc newContext*(j : JsonNode = nil): Context = + ## Create a new Context based on a JsonNode object + new(result) + if j == nil: + result = Context(kind: CObject) + result.fields = initTable[string, Context](4) + else: + case j.kind + of JObject: + result = Context(kind: CObject) + result.fields = initTable[string, Context](4) + for key, val in pairs(j.fields): + result.fields[key] = newContext(val) + of JArray: + result = Context(kind: CArray) + result.elems = @[] + for val in j.elems: + result.elems.add(newContext(val)) + else: + result = Context(kind: CValue) + result.val = j + +proc newContext*(c: Context): Context {.exportc.} = + ## Create a new Context based on an existing context. The new Context + ## is an unconnected copy of the existing context simply containing the + ## values of the original. + ## + ## Some code to demonstrate: + ## + ## .. code:: nim + ## + ## import moustachu + ## + ## var a = newContext() + ## a["test"] = "original" + ## + ## var b = a # copy the pointer to b + ## var c = newContext(a) # copy the content to c + ## + ## b["test"] = "changed" + ## + ## echo a["test"].toString() # -> "changed" + ## echo b["test"].toString() # -> "changed" + ## echo c["test"].toString() # -> "original" + new(result) + if c == nil: + result.kind = CObject + result.fields = initTable[string, Context](4) + else: + result.kind = c.kind + case c.kind + of CValue: + result.val = c.val + of CArray: + result.elems = @[] + for item in c.elems: + result.elems.add(newContext(item)) + of CObject: + result.fields = initTable[string, Context](4) + for key, val in pairs(c.fields): + result.fields[key] = newContext(val) + +proc newArrayContext*(): Context = + ## Create a new Context of kind CArray + result = Context(kind: CArray) + result.elems = @[] + +proc internal_set(value: string): Context = + newContext(newJString(value)) + +proc internal_set(value: int): Context = + newContext(newJInt(value)) + +proc internal_set(value: float): Context = + newContext(newJFloat(value)) + +proc internal_set(value: bool): Context = + newContext(newJBool(value)) + +## ## Getters + +proc `[]`*(c: Context, key: string): Context = + ## Return the Context associated with `key`. + ## If the Context at `key` does not exist, return nil. + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + if c.kind != CObject: return nil + if c.fields.hasKey(key): return c.fields[key] else: return nil + +proc `[]`*(c: Context, index: int): Context = + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + if c.kind != CArray: return nil else: return c.elems[index] + +## Setters + +proc `[]=`*(c: var Context, key: string, value: Context) = + ## Assign a context `value` to `key` in context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c.fields[key] = value + +proc `[]=`*(c: var Context, key: string, value: JsonNode) = + ## Convert and assign `value` to `key` in `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = newContext(value) + +proc `[]=`*(c: var Context; key: string, value: BiggestInt) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = newContext(newJInt(value)) + +proc `[]=`*(c: var Context; key: string, value: string) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = newContext(newJString(value)) + +proc `[]=`*(c: var Context; key: string, value: float) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = newContext(newJFloat(value)) + +proc `[]=`*(c: var Context; key: string, value: bool) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = newContext(newJBool(value)) + +proc `[]=`*(c: var Context, key: string, value: openarray[Context]) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + var contextList = newArrayContext() + for v in value: + contextList.elems.add(v) + c[key] = contextList + +proc `[]=`*(c: var Context, key: string, value: openarray[string]) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = map(value, proc(x: string): Context = newContext(newJString(x))) + +proc `[]=`*(c: var Context, key: string, value: openarray[int]) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = map(value, proc(x: int): Context = newContext(newJInt(x))) + +proc `[]=`*(c: var Context, key: string, value: openarray[float]) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = map(value, proc(x: float): Context = newContext(newJFloat(x))) + +proc `[]=`*(c: var Context, key: string, value: openarray[bool]) = + ## Assign `value` to `key` in Context `c` + assert(c != nil, "Context is nil. Did you forget to initialize with newContext()?") + assert(c.kind == CObject) + c[key] = map(value, proc(x: bool): Context = newContext(newJBool(x))) + +proc add*(c: Context, value: Context) = + ## Add 'value' object to array. + assert(c.kind == CArray) + c.elems.add(value) + +proc add*[T: string, int, float, bool](c: Context, value: T) = + ## Add 'value' to array. Reference later with dot substitution "{{.}}" + assert(c.kind == CArray) + c.elems.add(internal_set(value)) + +## Printers + +proc `$`*(c: Context): string = + ## Return a string representing the context. Useful for debugging + if c == nil: + result = "Context->nil" + return + result = "Context->[kind: " & $c.kind + case c.kind + of CValue: + if c.val.str == "": + result &= "\nval: " + else: + result &= "\nval: " & $c.val + of CArray: + if c.elems == @[]: + result &= "\nnot initialized" + else: + var strArray = map(c.elems, proc(c: Context): string = $c) + result &= "\nelems: [" & join(strArray, ", ") & "]" + of CObject: + var strArray : seq[string] = @[] + for key, val in pairs(c.fields): + strArray.add(key & ": " & $val) + result &= "\nfields: {" & join(strArray, ", ") & "}" + result &= "\n]" + +proc toString*(c: Context): string = + ## Return string representation of `c` relevant to mustache + if c != nil: + if c.kind == CValue: + case c.val.kind + of JString: + if c.val.str == "": + return "" + return c.val.str + of JFloat: + return c.val.fnum.formatFloat(ffDefault, -1) + of JInt: + return $c.val.num + of JNull: + return "" + of JBool: + return if c.val.bval: "true" else: "" + else: + return $c.val + else: + return $c + else: + return "" + +proc len*(c: Context): int = + if c.kind == CArray: + result = c.elems.len + elif c.kind == CObject: + result = c.fields.len + else: discard + +converter toBool*(c: Context): bool = + assert(c.kind == CValue) + case c.val.kind + of JBool: result = c.val.bval + of JNull: result = false + of JString: result = c.val.str != "" + else: result = true + +proc newContext*[T: string | int | bool | float](d: openarray[tuple[key: string, value: T ]]): Context = + ## Create a new Context based on an array of [string, T] tuples + ## + ## For example, you could do: + ## var c = NewContext({"x": 7, "y": -20}) + ## or, + ## var c = NewContext({"r": "apple", "b": "bike"}) + ## or, if you must: + ## var c = NewContext([("r", "apple"), ("b", "tunnel")]) + var c = newContext() + for entry in d: + c[entry.key] = entry.value + return c
diff --git a/libs/moustachu_tokenizer.nim b/libs/moustachu_tokenizer.nim @@ -0,0 +1,128 @@ +import strutils + +type + TokenType* {.pure.} = enum + rawtext, + escapedvariable, + unescapedvariable, + section, + invertedsection, + comment, + ender, + indenter + + Token* = tuple[tokentype: TokenType, value: string] + + +proc left_side_empty(tmplate: string, pivotindex: int): tuple[empty: bool, newlineindex: int] = + var ls_i = 0 + var ls_empty = false + var i = pivotindex - 1 + while i > -1 and tmplate[i] in {' ', '\t'}: dec(i) + if (i == -1) or (tmplate[i] == '\l'): + ls_i = i + ls_empty = true + return (empty: ls_empty, newlineindex: ls_i) + + +iterator tokenize*(tmplate: string): Token = + let opening = "{{" + var pos = 0 + + while pos < tmplate.len: + let originalpos = pos + var closing = "}}" + + # find start of tag + var opening_index = tmplate.find(opening, start=pos) + if opening_index == -1: + yield (tokentype: TokenType.rawtext, value: tmplate[pos..high(tmplate)]) + break + + #Check if the left side is empty + var left_side = left_side_empty(tmplate, opening_index) + var ls_empty = left_side.empty + var ls_i = left_side.newlineindex + + #Deal with text before tag + var beforetoken = (tokentype: TokenType.rawtext, value: "") + if opening_index > pos: + #safe bet for now + beforetoken.value = tmplate[pos..opening_index-1] + + pos = opening_index + opening.len + + if not (pos < tmplate.len): + yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..high(tmplate)]) + break + + #Determine TokenType + var tt = TokenType.escapedvariable + + case tmplate[pos] + of '!': + tt = TokenType.comment + pos += 1 + of '&': + tt = TokenType.unescapedvariable + pos += 1 + of '{': + tt = TokenType.unescapedvariable + pos += 1 + closing &= "}" + of '#': + tt = TokenType.section + pos += 1 + of '^': + tt = TokenType.invertedsection + pos += 1 + of '/': + tt = TokenType.ender + pos += 1 + else: + tt = TokenType.escapedvariable + + #find end of tag + var closingindex = tmplate.find(closing, start=pos) + if closingindex == -1: + if beforetoken.value != "": yield beforetoken + yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..pos-1]) + continue + + #Check if the right side is empty + var rs_i = 0 + var rs_empty = false + var i = 0 + if ls_empty: + i = closingindex + closing.len + while i < tmplate.len and tmplate[i] in {' ', '\t'}: inc(i) + if i == tmplate.len: + rs_i = i - 1 + rs_empty = true + elif tmplate[i] == '\c' and (i+1 < tmplate.len) and (tmplate[i+1] == '\l'): + rs_i = i + 1 + rs_empty = true + elif tmplate[i] == '\l': + rs_i = i + rs_empty = true + else: + discard + + if tt in [TokenType.comment, TokenType.section, + TokenType.invertedsection, TokenType.ender]: + # Standalone tokens + if rs_empty: + if beforetoken.value != "": + beforetoken.value = tmplate[originalpos..ls_i] + yield beforetoken + + yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) + pos = rs_i + 1 # remove new line of this line + else: + if beforetoken.value != "": yield beforetoken + yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) + pos = closingindex + closing.len + else: + if beforetoken.value != "": yield beforetoken + yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) + pos = closingindex + closing.len
diff --git a/libs/tiny_sqlite.nim b/libs/tiny_sqlite.nim @@ -0,0 +1,662 @@ +## .. include:: ./tiny_sqlite/private/documentation.rst + +import std / [options, typetraits, sequtils] +from tiny_sqlite / sqlite_wrapper as sqlite import nil +import tiny_sqlite / private / stmtcache + +when not declared(tupleLen): + import macros + macro tupleLen(typ: typedesc[tuple]): int = + let impl = getType(typ) + result = newIntlitNode(impl[1].len - 1) + +export options.get, options.isSome, options.isNone + +type + DbConnImpl = ref object + handle: sqlite.Sqlite3 ## The underlying SQLite3 handle + cache: StmtCache + + DbConn* = distinct DbConnImpl ## Encapsulates a database connection. + + SqlStatementImpl = ref object + handle: sqlite.Stmt + db: DbConn + + SqlStatement* = distinct SqlStatementImpl ## A prepared SQL statement. + + DbMode* = enum + dbRead, + dbReadWrite + + SqliteError* = object of CatchableError ## \ + ## Raised when whenever a database related error occurs. + ## Errors are typically a result of API misuse, + ## e.g trying to close an already closed database connection. + + DbValueKind* = enum ## \ + ## Enum of all possible value types in a SQLite database. + sqliteNull, + sqliteInteger, + sqliteReal, + sqliteText, + sqliteBlob + + DbValue* = object ## \ + ## Can represent any value in a SQLite database. + case kind*: DbValueKind + of sqliteInteger: + intVal*: int64 + of sqliteReal: + floatVal*: float64 + of sqliteText: + strVal*: string + of sqliteBlob: + blobVal*: seq[byte] + of sqliteNull: + discard + + Rc = cint + + ResultRow* = object + values: seq[DbValue] + columns: seq[string] + +const SqliteRcOk = [ sqlite.SQLITE_OK, sqlite.SQLITE_DONE, sqlite.SQLITE_ROW ] + + +# Forward declarations +proc isInTransaction*(db: DbConn): bool {.noSideEffect.} +proc isOpen*(db: DbConn): bool {.noSideEffect, inline.} + +template handle(db: DbConn): sqlite.Sqlite3 = DbConnImpl(db).handle +template handle(statement: SqlStatement): sqlite.Stmt = SqlStatementImpl(statement).handle +template db(statement: SqlStatement): DbConn = SqlStatementImpl(statement).db +template cache(db: DbConn): StmtCache = DbConnImpl(db).cache + +template hasCache(db: DbConn): bool = db.cache.capacity > 0 + +template assertCanUseDb(db: DbConn) = + doAssert (not DbConnImpl(db).isNil) and (not db.handle.isNil), "Database is closed" + +template assertCanUseStatement(statement: SqlStatement, busyOk: static[bool] = false) = + doAssert (not SqlStatementImpl(statement).isNil) and (not statement.handle.isNil), + "Statement cannot be used because it has already been finalized." + doAssert not statement.db.handle.isNil, + "Statement cannot be used because the database connection has been closed" + when not busyOk: + doAssert not sqlite.stmt_busy(statement.handle), + "Statement cannot be used while inside the 'all' iterator" + +proc newSqliteError(db: DbConn): ref SqliteError = + ## Raises a SqliteError exception. + (ref SqliteError)(msg: "sqlite error: " & $sqlite.errmsg(db.handle)) + +proc newSqliteError(msg: string): ref SqliteError = + ## Raises a SqliteError exception. + (ref SqliteError)(msg: msg) + +template checkRc(db: DbConn, rc: Rc) = + if rc notin SqliteRcOk: + raise newSqliteError(db) + +proc skipLeadingWhiteSpaceAndComments(sql: var cstring) = + let original = sql + + template `&+`(s: cstring, offset: int): cstring = + cast[cstring](cast[ByteAddress](sql) + offset) + + while true: + case sql[0] + of {' ', '\t', '\v', '\r', '\l', '\f'}: + sql = sql &+ 1 + of '-': + if sql[1] == '-': + sql = sql &+ 2 + while sql[0] != '\n': + sql = sql &+ 1 + if sql[0] == '\0': + return + sql = sql &+ 1 + else: + return; + of '/': + if sql[1] == '*': + sql = sql &+ 2 + while sql[0] != '*' or sql[1] != '/': + sql = sql &+ 1 + if sql[0] == '\0': + sql = original + return + sql = sql &+ 2 + else: + return; + else: + return + +# +# DbValue +# + +proc toDbValue*[T: Ordinal](val: T): DbValue = + ## Convert an ordinal value to a Dbvalue. + DbValue(kind: sqliteInteger, intVal: val.int64) + +proc toDbValue*[T: SomeFloat](val: T): DbValue = + ## Convert a float to a DbValue. + DbValue(kind: sqliteReal, floatVal: val) + +proc toDbValue*[T: string](val: T): DbValue = + ## Convert a string to a DbValue. + DbValue(kind: sqliteText, strVal: val) + +proc toDbValue*[T: seq[byte]](val: T): DbValue = + ## Convert a sequence of bytes to a DbValue. + DbValue(kind: sqliteBlob, blobVal: val) + +proc toDbValue*[T: Option](val: T): DbValue = + ## Convert an optional value to a DbValue. + if val.isNone: + DbValue(kind: sqliteNull) + else: + toDbValue(val.get) + +proc toDbValue*[T: type(nil)](val: T): DbValue = + ## Convert a nil literal to a DbValue. + DbValue(kind: sqliteNull) + +proc toDbValues*(values: varargs[DbValue, toDbValue]): seq[DbValue] = + ## Convert several values to a sequence of DbValue's. + runnableExamples: + doAssert toDbValues("string", 23) == @[toDbValue("string"), toDbValue(23)] + @values + +proc fromDbValue*(value: DbValue, T: typedesc[Ordinal]): T = + # Convert a DbValue to an ordinal. + value.intval.T + +proc fromDbValue*(value: DbValue, T: typedesc[SomeFloat]): float64 = + ## Convert a DbValue to a float. + value.floatVal + +proc fromDbValue*(value: DbValue, T: typedesc[string]): string = + ## Convert a DbValue to a string. + value.strVal + +proc fromDbValue*(value: DbValue, T: typedesc[seq[byte]]): seq[byte] = + ## Convert a DbValue to a sequence of bytes. + value.blobVal + +proc fromDbValue*[T](value: DbValue, _: typedesc[Option[T]]): Option[T] = + ## Convert a DbValue to an optional value. + if value.kind == sqliteNull: + none(T) + else: + some(value.fromDbValue(T)) + +proc fromDbValue*(value: DbValue, T: typedesc[DbValue]): T = + ## Special overload that simply return `value`. + ## The purpose of this overload is to do partial unpacking. + ## For example, if the type of one column in a result row is unknown, + ## the DbValue type can be kept just for that column. + ## + ## .. code-block:: nim + ## + ## for row in db.iterate("SELECT name, extra FROM Person"): + ## # Type of 'extra' is unknown, so we don't unpack it. + ## # The 'extra' variable will be of type 'DbValue' + ## let (name, extra) = row.unpack((string, DbValue)) + value + +proc `$`*(dbVal: DbValue): string = + result.add "DbValue[" + case dbVal.kind + of sqliteInteger: result.add $dbVal.intVal + of sqliteReal: result.add $dbVal.floatVal + of sqliteText: result.addQuoted dbVal.strVal + of sqliteBlob: result.add "<blob>" + of sqliteNull: result.add "nil" + result.add "]" + +proc `==`*(a, b: DbValue): bool = + ## Returns true if `a` and `b` represents the same value. + if a.kind != b.kind: + false + else: + case a.kind + of sqliteInteger: a.intVal == b.intVal + of sqliteReal: a.floatVal == b.floatVal + of sqliteText: a.strVal == b.strVal + of sqliteBlob: a.blobVal == b.blobVal + of sqliteNull: true + +# +# PStmt +# + +proc bindParams(db: DbConn, stmtHandle: sqlite.Stmt, params: varargs[DbValue]): Rc = + result = sqlite.SQLITE_OK + let expectedParamsLen = sqlite.bind_parameter_count(stmtHandle) + if expectedParamsLen != params.len: + raise newSqliteError("SQL statement contains " & $expectedParamsLen & + " parameters but only " & $params.len & " was provided.") + + var idx = 1'i32 + for value in params: + let rc = + case value.kind + of sqliteNull: + sqlite.bind_null(stmtHandle, idx) + of sqliteInteger: + sqlite.bind_int64(stmtHandle, idx, value.intval) + of sqliteReal: + sqlite.bind_double(stmtHandle, idx, value.floatVal) + of sqliteText: + sqlite.bind_text(stmtHandle, idx, value.strVal.cstring, value.strVal.len.int32, sqlite.SQLITE_TRANSIENT) + of sqliteBlob: + sqlite.bind_blob(stmtHandle, idx.int32, cast[string](value.blobVal).cstring, + value.blobVal.len.int32, sqlite.SQLITE_TRANSIENT) + + if rc notin SqliteRcOk: + return rc + idx.inc + +proc prepareSql(db: DbConn, sql: string): sqlite.Stmt = + var tail: cstring + let rc = sqlite.prepare_v2(db.handle, sql.cstring, sql.len.cint + 1, result, tail) + db.checkRc(rc) + tail.skipLeadingWhiteSpaceAndComments() + assert tail.len == 0, + "Only single SQL statement is allowed in this context. " & + "To execute several SQL statements, use 'execScript'" + +proc prepareSql(db: DbConn, sql: string, params: seq[DbValue]): sqlite.Stmt + {.raises: [SqliteError].} = + if db.hasCache: + result = db.cache.getOrDefault(sql) + if result.isNil: + result = prepareSql(db, sql) + db.cache[sql] = result + else: + result = prepareSql(db, sql) + let rc = db.bindParams(result, params) + db.checkRc(rc) + +proc readColumn(stmtHandle: sqlite.Stmt, col: int32): DbValue = + let columnType = sqlite.column_type(stmtHandle, col) + case columnType + of sqlite.SQLITE_INTEGER: + result = toDbValue(sqlite.column_int64(stmtHandle, col)) + of sqlite.SQLITE_FLOAT: + result = toDbValue(sqlite.column_double(stmtHandle, col)) + of sqlite.SQLITE_TEXT: + result = toDbValue($sqlite.column_text(stmtHandle, col)) + of sqlite.SQLITE_BLOB: + let blob = sqlite.column_blob(stmtHandle, col) + let bytes = sqlite.column_bytes(stmtHandle, col) + var s = newSeq[byte](bytes) + if bytes != 0: + copyMem(addr(s[0]), blob, bytes) + result = toDbValue(s) + of sqlite.SQLITE_NULL: + result = toDbValue(nil) + else: + raiseAssert "Unexpected column type: " & $columnType + +iterator iterate(db: DbConn, stmtOrHandle: sqlite.Stmt | SqlStatement, params: varargs[DbValue], + errorRc: var int32): ResultRow = + let stmtHandle = when stmtOrHandle is sqlite.Stmt: stmtOrHandle else: stmtOrHandle.handle + errorRc = db.bindParams(stmtHandle, params) + if errorRc in SqliteRcOk: + var rowLen = sqlite.column_count(stmtHandle) + var columns = newSeq[string](rowLen) + for idx in 0 ..< rowLen: + columns[idx] = $sqlite.column_name(stmtHandle, idx) + while true: + var row = ResultRow(values: newSeq[DbValue](rowLen), columns: columns) + when stmtOrHandle is sqlite.Stmt: + assertCanUseDb db + else: + assertCanUseStatement stmtOrHandle, busyOk = true + let rc = sqlite.step(stmtHandle) + if rc == sqlite.SQLITE_ROW: + for idx in 0 ..< rowLen: + row.values[idx] = readColumn(stmtHandle, idx) + yield row + elif rc == sqlite.SQLITE_DONE: + break + else: + errorRc = rc + break + +# +# DbConn +# + +proc exec*(db: DbConn, sql: string, params: varargs[DbValue, toDbValue]) = + ## Executes ``sql``, which must be a single SQL statement. + runnableExamples: + let db = openDatabase(":memory:") + db.exec("CREATE TABLE Person(name, age)") + db.exec("INSERT INTO Person(name, age) VALUES(?, ?)", + "John Doe", 23) + assertCanUseDb db + let stmtHandle = db.prepareSql(sql, @params) + let rc = sqlite.step(stmtHandle) + if db.hasCache: + discard sqlite.reset(stmtHandle) + else: + discard sqlite.finalize(stmtHandle) + db.checkRc(rc) + +template transaction*(db: DbConn, body: untyped) = + ## Starts a transaction and runs `body` within it. At the end the transaction is commited. + ## If an error is raised by `body` the transaction is rolled back. Nesting transactions is a no-op. + if db.isInTransaction: + body + else: + db.exec("BEGIN") + var ok = true + try: + try: + body + except Exception: + ok = false + db.exec("ROLLBACK") + raise + finally: + if ok: + db.exec("COMMIT") + +proc execMany*(db: DbConn, sql: string, params: seq[seq[DbValue]]) = + ## Executes ``sql``, which must be a single SQL statement, repeatedly using each element of + ## ``params`` as parameters. The statements are executed inside a transaction. + assertCanUseDb db + db.transaction: + for p in params: + db.exec(sql, p) + +proc execScript*(db: DbConn, sql: string) = + ## Executes ``sql``, which can consist of multiple SQL statements. + ## The statements are executed inside a transaction. + assertCanUseDb db + db.transaction: + var remaining = sql.cstring + while remaining.len > 0: + var tail: cstring + var stmtHandle: sqlite.Stmt + var rc = sqlite.prepare_v2(db.handle, remaining, -1, stmtHandle, tail) + db.checkRc(rc) + rc = sqlite.step(stmtHandle) + discard sqlite.finalize(stmtHandle) + db.checkRc(rc) + remaining = tail + remaining.skipLeadingWhiteSpaceAndComments() + +iterator iterate*(db: DbConn, sql: string, + params: varargs[DbValue, toDbValue]): ResultRow = + ## Executes ``sql``, which must be a single SQL statement, and yields each result row one by one. + assertCanUseDb db + let stmtHandle = db.prepareSql(sql, @params) + var errorRc: int32 + try: + for row in db.iterate(stmtHandle, params, errorRc): + yield row + finally: + # The database might have been closed while iterating, in which + # case we don't need to clean up the statement. + if not db.handle.isNil: + if db.hasCache: + discard sqlite.reset(stmtHandle) + else: + discard sqlite.finalize(stmtHandle) + db.checkRc(errorRc) + +proc all*(db: DbConn, sql: string, + params: varargs[DbValue, toDbValue]): seq[ResultRow] = + ## Executes ``sql``, which must be a single SQL statement, and returns all result rows. + for row in db.iterate(sql, params): + result.add row + +proc one*(db: DbConn, sql: string, + params: varargs[DbValue, toDbValue]): Option[ResultRow] = + ## Executes `sql`, which must be a single SQL statement, and returns the first result row. + ## Returns `none(seq[DbValue])` if the result was empty. + for row in db.iterate(sql, params): + return some(row) + +proc value*(db: DbConn, sql: string, + params: varargs[DbValue, toDbValue]): Option[DbValue] = + ## Executes `sql`, which must be a single SQL statement, and returns the first column of the first result row. + ## Returns `none(DbValue)` if the result was empty. + for row in db.iterate(sql, params): + return some(row.values[0]) + +proc close*(db: DbConn) = + ## Closes the database connection. This should be called once the connection will no longer be used + ## to avoid leaking memory. Closing an already closed database is a harmless no-op. + if not db.isOpen: + return + var stmtHandle = sqlite.next_stmt(db.handle, nil) + while not stmtHandle.isNil: + discard sqlite.finalize(stmtHandle) + stmtHandle = sqlite.next_stmt(db.handle, nil) + db.cache.clear() + let rc = sqlite.close(db.handle) + db.checkRc(rc) + DbConnImpl(db).handle = nil + +proc lastInsertRowId*(db: DbConn): int64 = + ## Get the row id of the last inserted row. + ## For tables with an integer primary key, + ## the row id will be the primary key. + ## + ## For more information, refer to the SQLite documentation + ## (https://www.sqlite.org/c3ref/last_insert_rowid.html). + assertCanUseDb db + sqlite.last_insert_rowid(db.handle) + +proc changes*(db: DbConn): int32 = + ## Get the number of changes triggered by the most recent INSERT, UPDATE or + ## DELETE statement. + ## + ## For more information, refer to the SQLite documentation + ## (https://www.sqlite.org/c3ref/changes.html). + assertCanUseDb db + sqlite.changes(db.handle) + +proc isReadonly*(db: DbConn): bool = + ## Returns true if ``db`` is in readonly mode. + runnableExamples: + let db = openDatabase(":memory:") + doAssert not db.isReadonly + let db2 = openDatabase(":memory:", dbRead) + doAssert db2.isReadonly + assertCanUseDb db + sqlite.db_readonly(db.handle, "main") == 1 + +proc isOpen*(db: DbConn): bool {.inline.} = + ## Returns true if `db` has been opened and not yet closed. + runnableExamples: + var db: DbConn + doAssert not db.isOpen + db = openDatabase(":memory:") + doAssert db.isOpen + db.close() + doAssert not db.isOpen + (not DbConnImpl(db).isNil) and (not db.handle.isNil) + +proc isInTransaction*(db: DbConn): bool = + ## Returns true if a transaction is currently active. + runnableExamples: + let db = openDatabase(":memory:") + doAssert not db.isInTransaction + db.transaction: + doAssert db.isInTransaction + assertCanUseDb db + sqlite.get_autocommit(db.handle) == 0 + +proc unsafeHandle*(db: DbConn): sqlite.Sqlite3 {.inline.} = + ## Returns the raw SQLite3 handle. This can be used to interact directly with the SQLite C API + ## with the `tiny_sqlite/sqlite_wrapper` module. Note that the handle should not be used after `db.close` has + ## been called as doing so would break memory safety. + assert not DbConnImpl(db).handle.isNil, "Database is closed" + DbConnImpl(db).handle + +# +# SqlStatement +# + +proc stmt*(db: DbConn, sql: string): SqlStatement = + ## Constructs a prepared statement from `sql`. + assertCanUseDb db + let handle = prepareSql(db, sql) + SqlStatementImpl(handle: handle, db: db).SqlStatement + +proc exec*(statement: SqlStatement, params: varargs[DbValue, toDbValue]) = + ## Executes `statement` with `params` as parameters. + assertCanUseStatement statement + var rc = statement.db.bindParams(statement.handle, params) + if rc notin SqliteRcOk: + discard sqlite.reset(statement.handle) + statement.db.checkRc(rc) + else: + rc = sqlite.step(statement.handle) + discard sqlite.reset(statement.handle) + statement.db.checkRc(rc) + +proc execMany*(statement: SqlStatement, params: seq[seq[DbValue]]) = + ## Executes ``statement`` repeatedly using each element of ``params`` as parameters. + ## The statements are executed inside a transaction. + assertCanUseStatement statement + statement.db.transaction: + for p in params: + statement.exec(p) + +iterator iterate*(statement: SqlStatement, params: varargs[DbValue, toDbValue]): ResultRow = + ## Executes ``statement`` and yields each result row one by one. + assertCanUseStatement statement + var errorRc: int32 + try: + for row in statement.db.iterate(statement, params, errorRc): + yield row + finally: + # The database might have been closed while iterating, in which + # case we don't need to clean up the statement. + if not statement.db.handle.isNil: + discard sqlite.reset(statement.handle) + statement.db.checkRc errorRc + +proc all*(statement: SqlStatement, params: varargs[DbValue, toDbValue]): seq[ResultRow] = + ## Executes ``statement`` and returns all result rows. + assertCanUseStatement statement + for row in statement.iterate(params): + result.add row + +proc one*(statement: SqlStatement, + params: varargs[DbValue, toDbValue]): Option[ResultRow] = + ## Executes `statement` and returns the first row found. + ## Returns `none(seq[DbValue])` if no result was found. + assertCanUseStatement statement + for row in statement.iterate(params): + return some(row) + +proc value*(statement: SqlStatement, + params: varargs[DbValue, toDbValue]): Option[DbValue] = + ## Executes `statement` and returns the first column of the first row found. + ## Returns `none(DbValue)` if no result was found. + assertCanUseStatement statement + for row in statement.iterate(params): + return some(row.values[0]) + +proc finalize*(statement: SqlStatement): void = + ## Finalize the statement. This needs to be called once the statement is no longer used to + ## prevent memory leaks. Finalizing an already finalized statement is a harmless no-op. + if SqlStatementImpl(statement).isNil: + return + discard sqlite.finalize(statement.handle) + SqlStatementImpl(statement).handle = nil + +proc isAlive*(statement: SqlStatement): bool = + ## Returns true if ``statement`` has been initialized and not yet finalized. + (not SqlStatementImpl(statement).isNil) and (not statement.handle.isNil) and + (not statement.db.handle.isNil) + +proc openDatabase*(path: string, mode = dbReadWrite, cacheSize: Natural = 100): DbConn = + ## Open a new database connection to a database file. To create an + ## in-memory database the special path `":memory:"` can be used. + ## If the database doesn't already exist and ``mode`` is ``dbReadWrite``, + ## the database will be created. If the database doesn't exist and ``mode`` + ## is ``dbRead``, a ``SqliteError`` exception will be raised. + ## + ## NOTE: To avoid memory leaks, ``db.close`` must be called when the + ## database connection is no longer needed. + runnableExamples: + let memDb = openDatabase(":memory:") + var handle: sqlite.Sqlite3 + let db = new DbConnImpl + db.handle = handle + if cacheSize > 0: + db.cache = initStmtCache(cacheSize) + result = DbConn(db) + case mode + of dbReadWrite: + let rc = sqlite.open(path, db.handle) + result.checkRc(rc) + of dbRead: + let rc = sqlite.open_v2(path, db.handle, sqlite.SQLITE_OPEN_READONLY, nil) + result.checkRc(rc) + +# +# ResultRow +# + +proc `[]`*(row: ResultRow, idx: Natural): DbValue = + ## Access a column in the result row based on index. + row.values[idx] + +proc `[]`*(row: ResultRow, column: string): DbValue = + ## Access a column in te result row based on column name. + ## The column name must be unambiguous. + let idx = row.columns.find(column) + assert idx != -1, "Column does not exist in row: '" & column & "'" + doAssert count(row.columns, column) == 1, "Column exists multiple times in row: '" & column & "'" + row.values[idx] + +proc len*(row: ResultRow): int = + ## Returns the number of columns in the result row. + row.values.len + +proc values*(row: ResultRow): seq[DbValue] = + ## Returns all column values in the result row. + row.values + +proc columns*(row: ResultRow): seq[string] = + ## Returns all column names in the result row. + row.columns + +proc unpack*[T: tuple](row: ResultRow, _: typedesc[T]): T = + ## Calls ``fromDbValue`` on each element of ``row`` and returns it + ## as a tuple. + doAssert row.len == result.typeof.tupleLen, + "Unpack expected a tuple with " & $row.len & " field(s) but found: " & $T + var idx = 0 + for value in result.fields: + value = row[idx].fromDbValue(type(value)) + idx.inc + +# +# Deprecations +# + +proc rows*(db: DbConn, sql: string, params: varargs[DbValue, toDbValue]): seq[seq[DbValue]] + {.deprecated: "use 'all' instead".} = + db.all(sql, params).mapIt(it.values) + +iterator rows*(db: DbConn, sql: string, params: varargs[DbValue, toDbValue]): seq[DbValue] + {.deprecated: "use 'iterate' instead".} = + for row in db.all(sql, params): + yield row.values + +proc unpack*[T: tuple](row: seq[DbValue], _: typedesc[T]): T {.deprecated.} = + ResultRow(values: row).unpack(T)+ \ No newline at end of file
diff --git a/libs/tiny_sqlite/private/documentation.rst b/libs/tiny_sqlite/private/documentation.rst @@ -0,0 +1,182 @@ +*********** +tiny_sqlite +*********** + +Opening a database connection. +############################## + +A database connection is opened by calling the `openDatabase <#openDatabase,string,Natural>`_ procedure with the +path to the database file as an argument. If the file doesn't exist, it will be created. An in-memory database can +be created by using the special path `":memory:"` as an argument. Once the database connection is no longer needed +`close <#close,DbConn>`_ must be called to prevent memory leaks. + +.. code-block:: nim + + let db = openDatabase("path/to/file.db") + # ... (do something with `db`) + db.close() + +Executing SQL +############# + +The `exec <#exec,DbConn,string,varargs[DbValue,toDbValue]>`_ procedure can be used to execute a single SQL statement. +The `execScript <#execScript,DbConn,string>`_ procedure is used to execute several statements, but it doesn't support +parameter substitution. + +.. code-block:: nim + + db.execScript(""" + CREATE TABLE Person( + name TEXT, + age INTEGER + ); + + CREATE TABLE Log( + message TEXT + ); + """) + + db.exec(""" + INSERT INTO Person(name, age) + VALUES(?, ?); + """, "John Doe", 37) + +Reading data +############ + +Four different procedures for reading data are available: + +- `all <#all,DbConn,string,varargs[DbValue,toDbValue]>`_: procedure returning all result rows +- `iterate <#iterate.i,DbConn,string,varargs[DbValue,toDbValue]>`_: iterator yielding each result row one by one +- `one <#one,DbConn,string,varargs[DbValue,toDbValue]>`_: procedure returning the first result row, or `none` if no result row exists +- `value <#value,DbConn,string,varargs[DbValue,toDbValue]>`_: procedure returning the first column of the first result row, or `none` if no result row exists + +Note that the procedures `one` and `value` returns the result wrapped in an `Option`. See the standard library +`options module <https://nim-lang.org/docs/options.html>`_ for documentation on how to deal with `Option` values. +For convenience the `tiny_sqlite` module exports the `options.get`, `options.isSome`, and `options.isNone` procedures so the options +module doesn't need to be explicitly imported for typical usage. + +.. code-block:: nim + + for row in db.iterate("SELECT name, age FROM Person"): + # The 'row' variable is of type ResultRow. + # The column values can be accesed by both index and column name: + echo row[0].strVal # Prints the name + echo row["name"].strVal # Prints the name + echo row[1].intVal # Prints the age + # Above we're using the raw DbValue's directly. Instead, we can unpack the + # DbValue using the fromDbValue procedure: + echo fromDbValue(row[0], string) # Prints the name + echo fromDbValue(row[1], int) # Prints the age + # Alternatively, the entire row can be unpacked at once: + let (name, age) = row.unpack((string, int)) + # Unpacking the value is preferable as it makes it possible to handle + # bools, enums, distinct types, nullable types and more. For example, nullable + # types are handled using Option[T]: + echo fromDbValue(row[0], Option[string]) # Will work even if the db value is NULL + + # Example of reading a single value. In this case, 'value' will be of type `Option[DbValue]`. + let value = db.one("SELECT age FROM Person WHERE name = ?", "John Doe") + if value.isSome: + echo fromDbValue(value.get, int) # Prints age of John Doe + + +Inserting data in bulk +###################### + +The `exec <#exec,DbConn,string,varargs[DbValue,toDbValue]>`_ procedure works fine for inserting single rows, +but it gets awkward when inserting many rows. For this purpose the `execMany <#execMany,DbConn,string,varargs[DbValue,toDbValue]>`_ +procedure can be used instead. It executes the same SQL repeatedly, but with different parameters each time. + +.. code-block:: nim + + let parameters = @[toDbValues("Person 1", 17), toDbValues("Person 2", 55)] + # Will insert two rows + db.execMany(""" + INSERT INTO Person(name, age) + VALUES(?, ?); + """, parameters) + +Transactions +############ + +The procedures that can execute multiple SQL statements (`execScript` and `execMany`) are wrapped in a transaction by +`tiny_sqlite`. Transactions can also be controlled manually by using one of these two options: + +- Option 1: using the `transaction <#transaction.t,DbConn,untyped>`_ template + +.. code-block:: nim + + db.transaction: + # Anything inside here is executed inside a transaction which + # will be rolled back in case of an error + db.exec("DELETE FROM Person") + db.exec("""INSERT INTO Person(name, age) VALUES("Jane Doe", 35)""") + +- Option 2: using the `exec` procedure manually + +.. code-block:: nim + + db.exec("BEGIN") + try: + db.exec("DELETE FROM Person") + db.exec("""INSERT INTO Person(name, age) VALUES("Jane Doe", 35)""") + db.exec("COMMIT") + except: + db.exec("ROLLBACK") + +Prepared statements +################### + +All the procedures for executing SQL described above create and execute prepared statements internally. In addition to +those procedures, ``tiny_sqlite`` also offers an API for preparing SQL statements explicitly. Prepared statements are +created with the `stmt <#stmt,DbConn,string>`_ procedure, and the same procedures for executing SQL that are available +directly on the connection object are also available for the prepared statement: + + +.. code-block:: nim + + let stmt = db.stmt("INSERT INTO Person(name, age) VALUES (?, ?)") + stmt.exec("John Doe", 21) + # Once the statement is no longer needed it must be finalized + # to prevent memory leaks. + stmt.finalize() + +There are performance benefits of reusing prepared statements, since the preparation only needs to be done once. +However, `tiny_sqlite` keeps an internal cache of prepared statements, so it's typically not necesarry to manage +prepared statements manually. If you prefer if `tiny_sqlite` doesn't perform this caching, you can disable it by +setting the `cacheSize` parameter when opening the database: + +.. code-block:: nim + + let db = openDatabase(":memory:", cacheSize = 0) + +Supported types +############### + +For a type to be supported when using unpacking and parameter substitution the procedures `toDbValue` and `fromDbValue` +must be implemented for the type. Below is table describing which types are supported by default and to which SQLite +type they are mapped to: + +==================== ================================================================================= +Nim type SQLite type +==================== ================================================================================= +``Ordinal`` | ``INTEGER`` +``SomeFloat`` | ``REAL`` +``string`` | ``TEXT`` +``seq[byte]`` | ``BLOB`` +``Option[T]`` | ``NULL`` if value is ``none(T)``, otherwise the type that ``T`` would use +==================== ================================================================================= + +This can be extended by implementing `toDdValue` and `fromDbValue` for other types on your own. Below is an example +how support for `times.Time` can be added: + +.. code-block:: nim + + import times + + proc toDbValue(t: Time): DbValue = + DbValue(kind: sqliteInteger, intVal: toUnix(t)) + + proc fromDbValue(value: DbValue, T: typedesc[Time]): Time = + fromUnix(value.intval)
diff --git a/libs/tiny_sqlite/private/stmtcache.nim b/libs/tiny_sqlite/private/stmtcache.nim @@ -0,0 +1,81 @@ +## Implements a least-recently-used cache for prepared statements based on +## https://github.com/jackhftang/lrucache.nim. + +import std / [lists, tables] +from .. / sqlite_wrapper as sqlite import nil + +type + Node = object + key: string + val: sqlite.Stmt + + StmtCache* = object + capacity: int + list: DoublyLinkedList[Node] + table: Table[string, DoublyLinkedNode[Node]] + +proc initStmtCache*(capacity: Natural): StmtCache = + ## Create a new Least-Recently-Used (LRU) cache that store the last `capacity`-accessed items. + StmtCache( + capacity: capacity, + list: initDoublyLinkedList[Node](), + table: initTable[string, DoublyLinkedNode[Node]](rightSize(capacity)) + ) + +proc resize(cache: var StmtCache) = + while cache.table.len > cache.capacity: + let t = cache.list.tail + cache.table.del(t.value.key) + discard sqlite.finalize(t.value.val) + cache.list.remove t + +proc capacity*(cache: StmtCache): int = + ## Get the maximum capacity of cache + cache.capacity + +proc len*(cache: StmtCache): int = + ## Return number of keys in cache + cache.table.len + +proc contains*(cache: StmtCache, key: string): bool = + ## Check whether key in cache. Does *NOT* update recentness. + cache.table.contains(key) + +proc clear*(cache: var StmtCache) = + ## remove all items + cache.list = initDoublyLinkedList[Node]() + cache.table.clear() + +proc `[]`*(cache: var StmtCache, key: string): sqlite.Stmt = + ## Read value from `cache` by `key` and update recentness + ## Raise `KeyError` if `key` is not in `cache`. + let node = cache.table[key] + result = node.value.val + cache.list.remove node + cache.list.prepend node + +proc `[]=`*(cache: var StmtCache, key: string, val: sqlite.Stmt) = + ## Put value `v` in cache with key `k`. + ## Remove least recently used value from cache if length exceeds capacity. + var node = cache.table.getOrDefault(key, nil) + if node.isNil: + let node = newDoublyLinkedNode[Node]( + Node(key: key, val: val) + ) + cache.table[key] = node + cache.list.prepend node + cache.resize() + else: + # set value + node.value.val = val + # move to head + cache.list.remove node + cache.list.prepend node + +proc getOrDefault*(cache: StmtCache, key: string, val: sqlite.Stmt = nil): sqlite.Stmt = + ## Similar to get, but return `val` if `key` is not in `cache` + let node = cache.table.getOrDefault(key, nil) + if node.isNil: + result = val + else: + result = node.value.val
diff --git a/libs/tiny_sqlite/sqlite_wrapper.nim b/libs/tiny_sqlite/sqlite_wrapper.nim @@ -0,0 +1,263 @@ +when defined(windows): + when defined(nimOldDlls): + const Lib = "sqlite3.dll" + elif defined(cpu64): + const Lib = "sqlite3_64.dll" + else: + const Lib = "sqlite3_32.dll" +elif defined(macosx): + const + Lib = "libsqlite3(|.0).dylib" +else: + const + Lib = "libsqlite3.so(|.0)" + +type + Sqlite3* = ptr object + + Stmt* = ptr object + + Callback* = proc (p: pointer, para2: cint, para3, para4: cstringArray): cint + {.cdecl, raises: [].} + + SqliteDestructor* = proc (p: pointer) + {.cdecl, locks: 0, tags: [], raises: [], gcsafe.} + +const + SQLITE_OK* = 0.cint + SQLITE_ERROR* = 1.cint # SQL error or missing database + SQLITE_INTERNAL* = 2.cint # An internal logic error in SQLite + SQLITE_PERM* = 3.cint # Access permission denied + SQLITE_ABORT* = 4.cint # Callback routine requested an abort + SQLITE_BUSY* = 5.cint # The database file is locked + SQLITE_LOCKED* = 6.cint # A table in the database is locked + SQLITE_NOMEM* = 7.cint # A malloc() failed + SQLITE_READONLY* = 8.cint # Attempt to write a readonly database + SQLITE_INTERRUPT* = 9.cint # Operation terminated by sqlite3_interrupt() + SQLITE_IOERR* = 10.cint # Some kind of disk I/O error occurred + SQLITE_CORRUPT* = 11.cint # The database disk image is malformed + SQLITE_NOTFOUND* = 12.cint # (Internal Only) Table or record not found + SQLITE_FULL* = 13.cint # Insertion failed because database is full + SQLITE_CANTOPEN* = 14.cint # Unable to open the database file + SQLITE_PROTOCOL* = 15.cint # Database lock protocol error + SQLITE_EMPTY* = 16.cint # Database is empty + SQLITE_SCHEMA* = 17.cint # The database schema changed + SQLITE_TOOBIG* = 18.cint # Too much data for one row of a table + SQLITE_CONSTRAINT* = 19.cint # Abort due to contraint violation + SQLITE_MISMATCH* = 20.cint # Data type mismatch + SQLITE_MISUSE* = 21.cint # Library used incorrectly + SQLITE_NOLFS* = 22.cint # Uses OS features not supported on host + SQLITE_AUTH* = 23.cint # Authorization denied + SQLITE_FORMAT* = 24.cint # Auxiliary database format error + SQLITE_RANGE* = 25.cint # 2nd parameter to sqlite3_bind out of range + SQLITE_NOTADB* = 26.cint # File opened that is not a database file + SQLITE_NOTICE* = 27.cint + SQLITE_WARNING* = 28.cint + SQLITE_ROW* = 100.cint # sqlite3_step() has another row ready + SQLITE_DONE* = 101.cint # sqlite3_step() has finished executing + +const + SQLITE_INTEGER* = 1.cint + SQLITE_FLOAT* = 2.cint + SQLITE_TEXT* = 3.cint + SQLITE_BLOB* = 4.cint + SQLITE_NULL* = 5.cint + SQLITE_UTF8* = 1.cint + SQLITE_UTF16LE* = 2.cint + SQLITE_UTF16BE* = 3.cint # Use native byte order + SQLITE_UTF16* = 4.cint # sqlite3_create_function only + SQLITE_ANY* = 5.cint #sqlite_exec return values + SQLITE_COPY* = 0.cint + SQLITE_CREATE_INDEX* = 1.cint + SQLITE_CREATE_TABLE* = 2.cint + SQLITE_CREATE_TEMP_INDEX* = 3.cint + SQLITE_CREATE_TEMP_TABLE* = 4.cint + SQLITE_CREATE_TEMP_TRIGGER* = 5.cint + SQLITE_CREATE_TEMP_VIEW* = 6.cint + SQLITE_CREATE_TRIGGER* = 7.cint + SQLITE_CREATE_VIEW* = 8.cint + SQLITE_DELETE* = 9.cint + SQLITE_DROP_INDEX* = 10.cint + SQLITE_DROP_TABLE* = 11.cint + SQLITE_DROP_TEMP_INDEX* = 12.cint + SQLITE_DROP_TEMP_TABLE* = 13.cint + SQLITE_DROP_TEMP_TRIGGER* = 14.cint + SQLITE_DROP_TEMP_VIEW* = 15.cint + SQLITE_DROP_TRIGGER* = 16.cint + SQLITE_DROP_VIEW* = 17.cint + SQLITE_INSERT* = 18.cint + SQLITE_PRAGMA* = 19.cint + SQLITE_READ* = 20.cint + SQLITE_SELECT* = 21.cint + SQLITE_TRANSACTION* = 22.cint + SQLITE_UPDATE* = 23.cint + SQLITE_ATTACH* = 24.cint + SQLITE_DETACH* = 25.cint + SQLITE_ALTER_TABLE* = 26.cint + SQLITE_REINDEX* = 27.cint + SQLITE_DENY* = 1.cint + SQLITE_IGNORE* = 2.cint + SQLITE_DETERMINISTIC* = 0x800.cint + +const + SQLITE_OPEN_READONLY* = 0x00000001.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_READWRITE* = 0x00000002.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_CREATE* = 0x00000004.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_DELETEONCLOSE* = 0x00000008.cint #/* VFS only */ + SQLITE_OPEN_EXCLUSIVE* = 0x00000010.cint #/* VFS only */ + SQLITE_OPEN_AUTOPROXY* = 0x00000020.cint #/* VFS only */ + SQLITE_OPEN_URI* = 0x00000040.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_MEMORY* = 0x00000080.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_MAIN_DB* = 0x00000100.cint #/* VFS only */ + SQLITE_OPEN_TEMP_DB* = 0x00000200.cint #/* VFS only */ + SQLITE_OPEN_TRANSIENT_DB* = 0x00000400.cint #/* VFS only */ + SQLITE_OPEN_MAIN_JOURNAL* = 0x00000800.cint #/* VFS only */ + SQLITE_OPEN_TEMP_JOURNAL* = 0x00001000.cint #/* VFS only */ + SQLITE_OPEN_SUBJOURNAL* = 0x00002000.cint #/* VFS only */ + SQLITE_OPEN_MASTER_JOURNAL* = 0x00004000.cint #/* VFS only */ + SQLITE_OPEN_NOMUTEX* = 0x00008000.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_FULLMUTEX* = 0x00010000.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_SHAREDCACHE* = 0x00020000.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_PRIVATECACHE* = 0x00040000.cint #/* Ok for sqlite3_open_v2() */ + SQLITE_OPEN_WAL* = 0x00080000.cint #/* VFS only */ + +const + SQLITE_STATIC* = nil + SQLITE_TRANSIENT* = cast[SqliteDestructor](-1) + +proc close*(db: Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_close".} + +proc exec*(db: Sqlite3, sql: cstring, cb: Callback, p: pointer, errmsg: var cstring): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_exec".} + +proc last_insert_rowid*(db: Sqlite3): int64 + {.cdecl, dynlib: Lib, importc: "sqlite3_last_insert_rowid".} + +proc changes*(db: Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_changes".} + +proc total_changes*(db: Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_total_changes".} + +proc busy_handler*(db: Sqlite3, + handler: proc (p: pointer, x: cint): cint {.cdecl.}, + p: pointer): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_busy_handler".} + +proc busy_timeout*(db: Sqlite3, ms: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_busy_timeout".} + +proc open*(filename: cstring, db: var Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_open".} + +proc open_v2*(filename: cstring, db: var Sqlite3, flags: cint, zVfsName: cstring ): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_open_v2".} + +proc errcode*(db: Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_errcode".} + +proc errmsg*(db: Sqlite3): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_errmsg".} + +proc prepare_v2*(db: Sqlite3, zSql: cstring, nByte: cint, stmt: var Stmt, + pzTail: var cstring): cint + {.importc: "sqlite3_prepare_v2", cdecl, dynlib: Lib.} + +proc bind_blob*(stmt: Stmt, col: cint, value: pointer, len: cint, + para5: SqliteDestructor): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_blob".} + +proc bind_double*(stmt: Stmt, col: cint, value: float64): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_double".} + +proc bind_int*(stmt: Stmt, col: cint, value: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_int".} + +proc bind_int64*(stmt: Stmt, col: cint, value: int64): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_int64".} + +proc bind_null*(stmt: Stmt, col: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_null".} + +proc bind_text*(stmt: Stmt, col: cint, value: cstring, len: cint, + destructor: SqliteDestructor): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_text".} + +proc bind_parameter_count*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_parameter_count".} + +proc bind_parameter_name*(stmt: Stmt, col: cint): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_parameter_name".} + +proc bind_parameter_index*(stmt: Stmt, colName: cstring): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_bind_parameter_index".} + +proc clear_bindings*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_clear_bindings".} + +proc column_count*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_column_count".} + +proc column_name*(stmt: Stmt, col: cint): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_column_name".} + +proc column_table_name*(stmt: Stmt, col: cint): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_column_table_name".} + +proc column_decltype*(stmt: Stmt, col: cint): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_column_decltype".} + +proc step*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_step".} + +proc data_count*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_data_count".} + +proc column_blob*(stmt: Stmt, col: cint): pointer + {.cdecl, dynlib: Lib, importc: "sqlite3_column_blob".} + +proc column_bytes*(stmt: Stmt, col: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_column_bytes".} + +proc column_double*(stmt: Stmt, col: cint): float64 + {.cdecl, dynlib: Lib, importc: "sqlite3_column_double".} + +proc column_int*(stmt: Stmt, col: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_column_int".} + +proc column_int64*(stmt: Stmt, col: cint): int64 + {.cdecl, dynlib: Lib, importc: "sqlite3_column_int64".} + +proc column_text*(stmt: Stmt, col: cint): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_column_text".} + +proc column_type*(stmt: Stmt, col: cint): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_column_type".} + +proc finalize*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_finalize".} + +proc reset*(stmt: Stmt): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_reset".} + +proc libversion*(): cstring + {.cdecl, dynlib: Lib, importc: "sqlite3_libversion".} + +proc libversion_number*(): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_libversion_number".} + +proc db_handle*(stmt: Stmt): Sqlite3 + {.cdecl, dynlib: Lib, importc: "sqlite3_db_handle".} + +proc get_autocommit*(db: Sqlite3): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_get_autocommit".} + +proc db_readonly*(db: Sqlite3, dbname: cstring): cint + {.cdecl, dynlib: Lib, importc: "sqlite3_db_readonly".} + +proc next_stmt*(db: Sqlite3, stmt: Stmt): Stmt + {.cdecl, dynlib: Lib, importc: "sqlite3_next_stmt".} + +proc stmt_busy*(stmt: Stmt): bool + {.cdecl, dynlib: Lib, importc: "sqlite3_stmt_busy".}+ \ No newline at end of file
diff --git a/mastofetch.nim b/mastofetch.nim @@ -0,0 +1,147 @@ +import std/[os, strutils, json, httpclient, times] +import libs/tiny_sqlite + +type + User = object + id* : int64 + username* : string + displayName* : string + url* : string + + Image = object + id* : int64 + url* : string + width* : int + height* : int + + Post = object + id* : int64 + url* : string + user* : User + createdAt* : DateTime + content* : string + attachedImages* : seq[Image] + +proc fetchPosts (client: HttpClient, instance: string, userId: int64, maxId: int64 = 0): seq[Post] = + var + url : string = "https://" & instance & "/api/v1/accounts/" & $userId & "/statuses" + response : string + + if maxId != 0: url.add("?max_id=" & $maxId) + + while true: + try: + response = client.getContent(url) + break + except HttpRequestError: + let msg = getCurrentExceptionMsg() + if msg == "429 Too Many Requests": + echo "Hit a rate-limit! I'll take a 30 sec break!" + sleep(30000) + else: + break + + let jsonData = response.parseJson + + for post in jsonData: + var images = newSeq[Image]() + + for image in post["media_attachments"]: + images.add(Image( + id : image["id"].getStr.parseInt, + url : image["url"].getStr, + width : image["meta"]["original"]["width"].getInt, + height : image["meta"]["original"]["height"].getInt + )) + + result.add(Post( + id : post["id"].getStr.parseInt, + url : post["url"].getStr, + user : User( + id : post["account"]["id"].getStr.parseInt, + username : post["account"]["username"].getStr, + displayName : post["account"]["display_name"].getStr, + url : post["account"]["url"].getStr, + ), + createdAt : parse(post["created_at"].getStr, "yyyy-MM-dd'T'HH:mm:ss'.'fff'Z'"), + content : post["content"].getStr, + attachedImages : images + )) + +proc postExists(db: DbConn, id: int64): bool = + let query = db.one("SELECT EXISTS(SELECT 1 FROM posts WHERE id=?);", id) + if query.isSome: + return fromDbValue(query.get[0], bool) + else: + return false + + +if getEnv("DB_PATH") == "": quit "No DB_PATH given!" + +let db = openDatabase(getEnv("DB_PATH")) + +db.execScript(""" +CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY, + url TEXT NOT NULL, + userId INTEGER NOT NULL, + userName TEXT NOT NULL, + userDisplay TEXT NOT NULL, + userUrl TEXT NOT NULL, + createdAt VACHAR(20) NOT NULL, + content TEXT +); + +CREATE TABLE IF NOT EXISTS images ( + id INTEGER PRIMARY KEY, + post_id INTEGER NOT NULL, + url TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + containsHorn BOOL NOT NULL +);""") + +let + client : HttpClient = newHttpClient() + instance : string = if getEnv("INSTANCE") != "": getEnv("INSTANCE") else: "chaos.social" + userId : int64 = if getEnv("USER_ID") != "": getEnv("USER_ID").parseInt else: 000069675 + oldestPostId : int64 = if getEnv("OLDEST_POST") != "": getEnv("OLDEST_POST").parseBiggestInt else: 101063382828117630 + allPostsFetched : bool = db.postExists(oldestPostId) + +var + posts : seq[Post] = client.fetchPosts(instance, userId) + lastPostId : int64 = 0 + +block fetchPosts: + while true: + for post in posts: + lastPostId = post.id + + if not db.postExists(post.id): + db.exec(""" + INSERT INTO posts(id, url, userId, userName, userDisplay, userUrl, createdAt, content) + VALUES(?, ?, ?, ?, ?, ?, ?, ?); + """, post.id, post.url, post.user.id, post.user.username, post.user.displayName, post.user.url, post.createdAt.format("yyyy-MM-dd'T'HH:mm:ss"), post.content) + + for image in post.attachedImages: + db.exec(""" + INSERT INTO images(id, post_id, url, width, height, containsHorn) + VALUES(?, ?, ?, ?, ?, ?); + """, image.id, post.id, image.url, image.width, image.height, true) + + echo "Inserted new post with id: " & $post.id + + if post.id == oldestPostId: + echo "All posts successfully fetched!" + break fetchPosts + + else: + if allPostsFetched: + echo "All new posts fetched!" + break fetchPosts + else: + echo "Post exists!" + + posts = client.fetchPosts(instance, userId, lastPostId) + +db.close()
diff --git a/website.nim b/website.nim @@ -0,0 +1,41 @@ +import std/[os, options, cgi, times, strutils] +import libs/tiny_sqlite +import libs/[moustachu, moustachu_context] + +write(stdout, "Content-Type: text/html; charset=UTF-8\n") +write(stdout, "Cache-Control: no-store, must-revalidate\n") +write(stdout, "Pragma: no-cache\n") +write(stdout, "Expires: 0\n") +write(stdout, "\n") + +if getEnv("DB_PATH") == "": + write(stdout, "No DB_PATH given!") + quit(QuitSuccess) + +const websiteTemplate : string = staticRead "website.tpl" +var templateContext : Context = newContext() +let + db : DbConn = openDatabase(getEnv("DB_PATH")) + image : Option[ResultRow] = db.one("SELECT post_id, url FROM images WHERE containsHorn = 1 ORDER BY RANDOM() LIMIT 1;") + + +templateContext["SCRIPT_URL"] = getRequestURI() + +if image.isSome: + templateContext["IMAGE_URL"] = fromDbValue(image.get["url"], string) + + let post : Option[ResultRow] = db.one("SELECT url, content, createdAt FROM posts WHERE id = ? LIMIT 1;", fromDbValue(image.get["post_id"], int64)) + + if post.isSome: + let postDateTime : DateTime = fromDbValue(post.get["createdAt"], string).parse("yyyy-MM-dd'T'HH:mm:ss") + + templateContext["POST_CONTENT"] = fromDbValue(post.get["content"], string).replace("<p>", "").replace("</p>", "") + templateContext["POST_URL"] = fromDbValue(post.get["url"], string) + templateContext["POST_DATE"] = postDateTime.format("dd'.'MM'.'yyyy") + templateContext["POST_TIME"] = postDateTime.format("HH:mm") +else: + write(stdout, "Database seems empty! :(") + quit(QuitSuccess) + +db.close() +write(stdout, render(websiteTemplate, templateContext))
diff --git a/website.tpl b/website.tpl @@ -0,0 +1,49 @@ +<html> + <head> + <title>flauschehorn.sexy</title> + <meta name=viewport content="width=device-width, initial-scale=1"> + <style> + html { + background-color: black; + } + + img { + position: absolute; + margin: auto; + top: 0; + left: 0; + right: 0; + bottom: 0; + height: 95%; + } + + p { + position: fixed; + bottom: 0; + left: 50%; + transform: translate(-50%, -50%); + background-color: rgba(255, 255, 255, 0.8); + padding: 10px; + font-family: sans-serif; + } + + @media only screen and (max-width: 600px) { + img { + height: unset; + width: 98%; + } + + p { + left: 0; + right 0; + transform: unset; + margin:0; + } + } + </style> + </head> + <body> + <a href="{{SCRIPT_URL}}"><img src="{{IMAGE_URL}}"></a> + <p>{{POST_CONTENT}} <a href="{{POST_URL}}" target="_blank">{{POST_DATE}}, {{POST_TIME}} Uhr</a></p> + </body> +</html>