/*
 * asklyphe-common db/mod.rs
 * - foundationdb functions for accessing crawled websites
 *
 * Copyright (C) 2025 Real Microsoft, LLC
 *
 * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use chrono::{TimeZone};
use foundationdb::{Database, RangeOption};
use foundationdb::options::StreamingMode;
use foundationdb::tuple::{pack, Subspace, unpack};
use log::{error, warn};
use rand::Rng;
use crate::db::tables::{DEHOMOGENISE_TABLE, HASH_TABLE, HOST_TABLE, LINK_TABLE, LINKWORD_TABLE, METAWORD_TABLE, WEBSITE_TABLE, WORD_TABLE, INFO_TABLE, REVLINK_TABLE, WEBSITE_TABLE_HOST, WEBSITE_TABLE_DAMPING, WEBSITE_TABLE_LASTPARSEBEGIN, WEBSITE_TABLE_LASTPARSEFINISH, WEBSITE_TABLE_LASTPARSEFINISHTRUE, WEBSITE_TABLE_CACHEDPAGERANK, WEBSITE_TABLE_LASTPAGERANKCACHE, WEBSITE_TABLE_ROBOTSFLAG, INFO_TABLE_TITLE, INFO_TABLE_DESC, INFO_TABLE_FAVICON, METAWURL_TABLE, WURL_TABLE, WEBSITE_TABLE_WORDCOUNT, DERANK_TABLE};


pub mod tables;

pub type STRHASH = i64;

pub fn hash(val: &str) -> STRHASH {
    let mut hasher = DefaultHasher::new();
    val.hash(&mut hasher);
    unsafe { std::mem::transmute(hasher.finish()) }
}

pub struct DerankEntry {
    pub urlmatch: String,
    pub and: Option<String>,
    pub unless: Option<String>,
    pub amount: f64,
    pub comment: String,
}

pub async fn remove_deranked_website(db: &Database, id: i64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        eprintln!("DATABASE ERROR remove_deranked_website {e}");
    } else {
        let trx = trx.unwrap();
        // info -> url
        let subspace = Subspace::from(id);

        trx.clear(&subspace.pack(&id));

        if let Err(e) = trx.commit().await {
            eprintln!("DATABASE ERROR remove_deranked_website {e}");
        }
    }
}

pub async fn add_deranked_website(db: &Database, entry: DerankEntry) -> Option<i64> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        eprintln!("DATABASE ERROR add_deranked_website {e}");
        None
    } else {
        let trx = trx.unwrap();
        let id: i64 = {
            let mut r = rand::thread_rng();
            r.gen()
        };
        // info -> url
        let subspace = Subspace::from(DERANK_TABLE);

        trx.set(&subspace.pack(&id), &pack(&(
            entry.urlmatch, entry.unless.clone().unwrap_or_default(), entry.and.clone().unwrap_or_default(),
            entry.unless.is_some(), entry.and.is_some(), entry.amount, entry.comment
        )));

        if let Err(e) = trx.commit().await {
            eprintln!("DATABASE ERROR add_deranked_website {e}");
        }

        Some(id)
    }
}

pub async fn get_deranked_websites(db: &Database) -> Vec<(i64, DerankEntry)> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR get_deranked_websites {e}");
        vec![]
    } else {
        let trx = trx.unwrap();
        // derank -> *
        let subspace = Subspace::from(DERANK_TABLE);
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR page_links_entering {e}");
            vec![]
        } else {
            let kv = kv.unwrap();

            let mut hashes = vec![];

            for kv in kv.into_iter() {
                let (_, id) = unpack::<(String, i64)>(kv.key()).expect("INVALID K TYPE get_deranked_websites");
                let (urlmatch, unless, and, use_unless, use_and, amount, comment)
                    = unpack::<
                    (String, String, String, bool, bool, f64, String)
                >(kv.value()).expect("INVALID V TYPE get_deranked_websites");

                hashes.push((id, DerankEntry {
                    urlmatch,
                    and: if use_and { Some(and) } else { None },
                    unless: if use_unless { Some(unless) } else { None },
                    amount,
                    comment,
                }));
            }

            hashes
        }
    }
}

pub async fn info_title(db: &Database, url: STRHASH) -> Option<String> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_pagerank {e}");
        None
    } else {
        let trx = trx.unwrap();
        // info -> url -> title
        let subspace = Subspace::from((INFO_TABLE, url));
        let v = trx.get(&subspace.pack(&(INFO_TABLE_TITLE)), true).await;

        if let Ok(v) = v {
            v.map(|v| unpack::<(String,)>(&v).expect("INVALID K TYPE page_pagerank").0)
        } else {
            None
        }
    }
}

pub async fn info_description(db: &Database, url: STRHASH) -> Option<String> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_pagerank {e}");
        None
    } else {
        let trx = trx.unwrap();
        // info -> url -> title
        let subspace = Subspace::from((INFO_TABLE, url));
        let v = trx.get(&subspace.pack(&(INFO_TABLE_DESC)), true).await;

        if let Ok(v) = v {
            v.map(|v| unpack::<(String,)>(&v).expect("INVALID K TYPE page_pagerank").0)
        } else {
            None
        }
    }
}

pub async fn set_page_word_count(db: &Database, url: STRHASH, wc: u64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR set_page_word_count {e}");
    } else {
        let trx = trx.unwrap();
        trx.set(&pack(&(WEBSITE_TABLE, url, WEBSITE_TABLE_WORDCOUNT)), &pack(&(wc)));
        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR set_page_word_count {e}");
        }
    }
}

pub async fn page_word_count(db: &Database, url: STRHASH) -> u64 {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_word_count {e}");
        0
    } else {
        let trx = trx.unwrap();

        let wc = trx.get(&pack(&(WEBSITE_TABLE, url, WEBSITE_TABLE_WORDCOUNT)), true).await;
        if let Ok(Some(wc)) = wc {
            let val = unpack::<u64>(&wc).unwrap();
            if val > 0 {
                return val;
            }
        } else if let Err(e) = wc {
            warn!("DATABASE ERROR page_word_count {e}");
        }
        warn!("recalculating page word count...");
        // link -> from -> *
        let subspace = Subspace::from((WURL_TABLE, url));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;


        if let Err(e) = kv {
            error!("DATABASE ERROR page_word_count {e}");
            0
        } else {
            let kv = kv.unwrap();
            let mut counter = 0;
            for kv in kv.into_iter() {
                let trx = db.create_trx();
                if let Err(e) = trx {
                    error!("DATABASE ERROR page_word_count {e}");
                } else {
                    let trx = trx.unwrap();
                    if let Ok(word) = unpack::<STRHASH>(kv.value()) {
                        let subspace = Subspace::from((WORD_TABLE, word));
                        let kv = trx.get(&subspace.pack(&url), true).await;
                        if let Err(e) = kv {
                            error!("DATABASE ERROR page_word_count {e}");
                        } else {
                            let kv = kv.unwrap();
                            if kv.is_none() {
                                warn!("WURL HAS NO CORRESPONDING WORD");
                            } else {
                                let kv = kv.unwrap();
                                counter += u64::from_be_bytes(kv.split_at(std::mem::size_of::<u64>()).0.try_into().unwrap());
                            }
                        }
                    }
                }
            }

            let add_wc_trx = db.create_trx();
            if let Err(e) = add_wc_trx {
                error!("couldn't add precalced wordcount to db: {e}");
            } else {
                let add_wc_trx = add_wc_trx.unwrap();
                add_wc_trx.set(&pack(&(WEBSITE_TABLE, url, WEBSITE_TABLE_WORDCOUNT)), &pack(&(counter)));
                if let Err(e) = add_wc_trx.commit().await {
                    warn!("couldn't add precalced wordcount to db: {e}");
                }
            }
            counter
        }
    }
}

pub async fn page_links_entering(db: &Database, url: STRHASH) -> Vec<STRHASH> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_links_entering {e}");
        vec![]
    } else {
        let trx = trx.unwrap();
        // revlink -> to -> *
        let subspace = Subspace::from((REVLINK_TABLE, url));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR page_links_entering {e}");
            vec![]
        } else {
            let kv = kv.unwrap();

            let mut hashes = vec![];

            for kv in kv.into_iter() {
                let (_, _, from) = unpack::<(String, i64, i64)>(kv.key()).expect("INVALID K TYPE page_links_entering");

                hashes.push(from);
            }

            hashes
        }
    }
}

pub async fn page_pagerank(db: &Database, url: STRHASH) -> Option<f64> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_pagerank {e}");
        None
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));
        let v = trx.get(&subspace.pack(&(WEBSITE_TABLE_CACHEDPAGERANK)), true).await;

        if let Ok(v) = v {
            v.map(|v| unpack::<(f64, )>(&v).expect("INVALID K TYPE page_pagerank").0)
        } else {
            None
        }
    }
}

pub async fn page_damping(db: &Database, url: STRHASH) -> Option<f64> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_damping {e}");
        None
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));
        let v = trx.get(&subspace.pack(&(WEBSITE_TABLE_DAMPING)), true).await;

        if let Ok(v) = v {
            v.map(|v| unpack::<(f64, )>(&v).expect("INVALID K TYPE page_damping").0)
        } else {
            None
        }
    }
}

pub async fn page_links_exiting_count(db: &Database, url: STRHASH) -> u64 {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR page_links_exiting_count {e}");
        0
    } else {
        let trx = trx.unwrap();
        // link -> from -> *
        let subspace = Subspace::from((LINK_TABLE, url));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;


        if let Err(e) = kv {
            error!("DATABASE ERROR page_links_exiting_count {e}");
            0
        } else {
            let kv = kv.unwrap();

            kv.len() as u64
        }
    }
}

// todo: precalc statements
// url, occurs
pub async fn word_occurs(db: &Database, word: STRHASH) -> Vec<(STRHASH, Vec<u8>)> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR word_occurs {e}");
        vec![]
    } else {
        let trx = trx.unwrap();
        // word -> word -> *
        let subspace = Subspace::from((WORD_TABLE, word));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR word_occurs {e}");
            vec![]
        } else {
            let kv = kv.unwrap();

            let mut results = vec![];

            for kv in kv.into_iter() {
                let (_, _, url) = unpack::<(String, STRHASH, STRHASH)>(kv.key()).expect("INVALID K TYPE word_occurs");

                results.push((url, kv.value().to_vec()));
            }

            results
        }
    }
}

// todo: precalc statements
// url, property, occurs
pub async fn metaword_occurs(db: &Database, property: STRHASH, word: STRHASH) -> Vec<(STRHASH, Vec<u8>)> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR metaword_occurs {e}");
        vec![]
    } else {
        let trx = trx.unwrap();
        // metaword -> word -> prop -> *
        let subspace = Subspace::from((METAWORD_TABLE, word, property));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR metaword_occurs {e}");
            vec![]
        } else {
            let kv = kv.unwrap();

            let mut results = vec![];

            for kv in kv.into_iter() {
                let (_, _, _, url) = unpack::<(String, STRHASH, STRHASH, STRHASH)>(kv.key()).expect("INVALID K TYPE metaword_occurs");

                results.push((url, kv.value().to_vec()));
            }

            results
        }
    }
}

// todo: precalc statements
// url, occurs
pub async fn linkword_occurs(db: &Database, word: STRHASH) -> Vec<(STRHASH, Vec<u8>)> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR linkword_occurs {e}");
        vec![]
    } else {
        let trx = trx.unwrap();
        // metaword -> word -> prop -> *
        let subspace = Subspace::from((LINKWORD_TABLE, word));
        let mut range = RangeOption::from(&subspace);
        range.mode = StreamingMode::WantAll;

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR linkword_occurs {e}");
            vec![]
        } else {
            let kv = kv.unwrap();

            let mut results = vec![];

            for kv in kv.into_iter() {
                let (_, _, _, url) = unpack::<(String, STRHASH, STRHASH, STRHASH)>(kv.key()).expect("INVALID K TYPE linkword_occurs");

                results.push((url, kv.value().to_vec()));
            }

            results
        }
    }
}

pub async fn foa_strhash(db: &Database, value: &str) -> STRHASH {
    let trx = db.create_trx();
    if let Err(e) = trx {
        panic!("FATAL DATABASE ERROR foa_strhash {e}");
    } else {
        let trx = trx.unwrap();
        // hash -> value
        let subspace = Subspace::from((HASH_TABLE, ));

        let hash = hash(value);

        trx.set(&subspace.pack(&hash), &pack(&(value.to_string(), )));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR foa_strhash (HASH MAY NOT HAVE BEEN INSERTED! {e}");
        }

        hash
    }
}

pub async fn fof_strhash(db: &Database, value: &str) -> Option<STRHASH> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR fof_strhash {e}");
        None
    } else {
        let trx = trx.unwrap();
        // hash -> value
        let subspace = Subspace::from((HASH_TABLE, ));

        let hash = hash(value);

        let v = trx.get(&subspace.pack(&hash), true).await;

        if let Err(e) = v {
            error!("DATABASE ERROR fof_strhash {e}");
            None
        } else {
            let v = v.unwrap();
            v.map(|_| hash)
        }
    }
}

pub async fn unhash(db: &Database, value: STRHASH) -> String {
    let trx = db.create_trx();
    if let Err(e) = trx {
        panic!("FATAL DATABASE ERROR unhash {e}");
    } else {
        let trx = trx.unwrap();
        // hash -> value
        let subspace = Subspace::from((HASH_TABLE, ));

        let v = trx.get(&subspace.pack(&value), true).await;

        if let Err(e) = v {
            panic!("FATAL DATABASE ERROR unhash {e}");
        } else {
            let v = v.unwrap();
            if v.is_none() {
                panic!("UNHASH CALLED ON NON-EXISTANT HASH {value}")
            }
            unpack::<String>(&v.unwrap()).expect("INVALID K TYPE unhash")
        }
    }
}

pub async fn host_in_database(db: &Database, url: STRHASH) -> bool {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR host_in_database {e}");
        false
    } else {
        let trx = trx.unwrap();
        // host -> url
        let subspace = Subspace::from(HOST_TABLE);

        let v = trx.get(&subspace.pack(&url), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR host_in_database {e}");
            false
        } else {
            v.unwrap().is_some()
        }
    }
}

pub async fn create_host(db: &Database, url: STRHASH) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR create_host {e}");
    } else {
        let trx = trx.unwrap();
        // host -> url
        let subspace = Subspace::from(HOST_TABLE);

        trx.set(&subspace.pack(&url), &pack(&(0i64)));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR create_host {e}");
        }
    }
}

pub async fn create_empty_document(db: &Database, url: STRHASH, host_url: STRHASH, damping: f32, parse_begin: Option<i64>) {
    if !document_in_database(db, url).await {
        let trx = db.create_trx();
        if let Err(e) = trx {
            error!("DATABASE ERROR create_empty_document {e}");
        } else {
            let trx = trx.unwrap();
            // website -> url
            let subspace = Subspace::from((WEBSITE_TABLE, url));

            trx.set(&subspace.pack(&WEBSITE_TABLE_HOST), &pack(&host_url));
            trx.set(&subspace.pack(&WEBSITE_TABLE_DAMPING), &pack(&(damping as f64)));
            trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEBEGIN), &pack(&parse_begin.unwrap_or(0i64)));
            trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEFINISH), &pack(&0i64));
            trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEFINISHTRUE), &pack(&0i64));
            trx.set(&subspace.pack(&WEBSITE_TABLE_CACHEDPAGERANK), &pack(&0f64));
            trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPAGERANKCACHE), &pack(&0i64));
            trx.set(&subspace.pack(&WEBSITE_TABLE_ROBOTSFLAG), &pack(&false));

            if let Err(e) = trx.commit().await {
                error!("DATABASE ERROR create_empty_document {e}");
            }
        }
    } else {}
}

pub async fn link_in_database(db: &Database, url: STRHASH, dest: STRHASH) -> bool {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR link_in_database {e}");
        false
    } else {
        let trx = trx.unwrap();
        // link -> from (url) -> to (dest)
        let subspace = Subspace::from((LINK_TABLE, url));

        let v = trx.get(&subspace.pack(&dest), true).await;

        if let Err(e) = v {
            error!("DATABASE ERROR link_in_database {e}");
            false
        } else {
            v.unwrap().is_some()
        }
    }
}

// creates an empty link between two websites, does nothing if it already exists
pub async fn create_empty_link(db: &Database, url: STRHASH, host: STRHASH, dest_url: STRHASH) {
    if !document_in_database(db.clone(), dest_url).await {
        create_empty_document(db.clone(), dest_url, host, 0.85, None).await;
    }
    if !link_in_database(db, url, dest_url).await {
        let trx = db.create_trx();
        if let Err(e) = trx {
            error!("DATABASE ERROR create_empty_link {e}");
        } else {
            let trx = trx.unwrap();
            // link -> from -> to && revlink -> to -> from
            let subspace = Subspace::from((LINK_TABLE, url));
            let revsubspace = Subspace::from((REVLINK_TABLE, dest_url));

            trx.set(&subspace.pack(&dest_url), &[0u8]);
            trx.set(&revsubspace.pack(&url), &[0u8]);

            if let Err(e) = trx.commit().await {
                error!("DATABASE ERROR create_empty_link {e}");
            }
        }
    }
}

pub async fn link_set_flags(db: &Database, url: STRHASH, dest_url: STRHASH, flags: u8) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR link_set_flags {e}");
    } else {
        let trx = trx.unwrap();
        // link -> from -> to && revlink -> to -> from
        let subspace = Subspace::from((LINK_TABLE, url));
        let revsubspace = Subspace::from((REVLINK_TABLE, dest_url));

        trx.set(&subspace.pack(&dest_url), &[flags]);
        trx.set(&revsubspace.pack(&url), &[flags]);

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR link_set_flags {e}");
        }
    }
}

pub async fn document_in_database(db: &Database, url: STRHASH) -> bool {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_in_database {e}");
        false
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        let v = trx.get(&subspace.pack(&WEBSITE_TABLE_HOST), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR document_in_database {e}");
            false
        } else {
            v.unwrap().is_some()
        }
    }
}

pub async fn info_in_database(db: &Database, url: STRHASH) -> bool {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR info_in_database {e}");
        false
    } else {
        let trx = trx.unwrap();
        // info -> url
        let subspace = Subspace::from(INFO_TABLE);

        let v = trx.get(&subspace.pack(&url), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR info_in_database {e}");
            false
        } else {
            v.unwrap().is_some()
        }
    }
}

pub async fn set_info(db: &Database, url: STRHASH, title: String, description: String, favicon: i64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR set_info {e}");
    } else {
        let trx = trx.unwrap();
        // info -> url
        let subspace = Subspace::from((INFO_TABLE, url));

        trx.set(&subspace.pack(&INFO_TABLE_TITLE), &pack(&title));
        trx.set(&subspace.pack(&INFO_TABLE_DESC), &pack(&(description)));
        trx.set(&subspace.pack(&INFO_TABLE_FAVICON), &pack(&favicon));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR set_info {e}");
        }
    }
}

pub async fn document_last_parse_begin(db: &Database, url: STRHASH) -> Option<chrono::DateTime<chrono::Utc>> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_last_parse_begin {e}");
        None
    } else {
        let trx = trx.unwrap();
        // website -> url -> lastparsebegin
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        let v = trx.get(&subspace.pack(&WEBSITE_TABLE_LASTPARSEBEGIN), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR document_last_parse_begin {e}");
            None
        } else {
            v.unwrap().map(|v| chrono::Utc.timestamp_opt(unpack::<i64>(&v).expect("INVALID K TYPE document_last_parse_begin"), 0).unwrap())
        }
    }
}

pub async fn document_last_parse_finish(db: &Database, url: STRHASH) -> Option<chrono::DateTime<chrono::Utc>> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_last_parse_finish {e}");
        None
    } else {
        let trx = trx.unwrap();
        // website -> url -> lastparsefinish
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        let v = trx.get(&subspace.pack(&WEBSITE_TABLE_LASTPARSEFINISH), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR document_last_parse_finish {e}");
            None
        } else {
            v.unwrap().map(|v| chrono::Utc.timestamp_opt(unpack::<i64>(&v).expect("INVALID K TYPE document_last_parse_finish"), 0).unwrap())
        }
    }
}

pub async fn document_set_damping(db: &Database, url: STRHASH, damping: f32) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_damping {e}");
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        trx.set(&subspace.pack(&WEBSITE_TABLE_DAMPING), &pack(&(damping as f64)));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_damping {e}");
        }
    }
}

pub async fn document_set_pagerank(db: &Database, url: STRHASH, pagerank: f64, now: i64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_pagerank {e}");
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        trx.set(&subspace.pack(&WEBSITE_TABLE_CACHEDPAGERANK), &pack(&(pagerank)));
        trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPAGERANKCACHE), &pack(&(now)));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_dampingdocument_set_pagerank {e}");
        }
    }
}

pub async fn document_set_last_parse_begin(db: &Database, url: STRHASH, now: i64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_last_parse_begin {e}");
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEBEGIN), &pack(&now));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_last_parse_begin {e}");
        }
    }
}

pub async fn document_set_last_parse_finish(db: &Database, url: STRHASH, now: i64, true_now: i64) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_last_parse_finish {e}");
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEFINISH), &pack(&now));
        trx.set(&subspace.pack(&WEBSITE_TABLE_LASTPARSEFINISHTRUE), &pack(&true_now));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_last_parse_finish {e}");
        }
    }
}

pub async fn document_set_robots_flag(db: &Database, url: STRHASH, value: bool) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_robots_flag {e}");
    } else {
        let trx = trx.unwrap();
        // website -> url
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        trx.set(&subspace.pack(&WEBSITE_TABLE_ROBOTSFLAG), &pack(&value));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_robots_flag {e}");
        }
    }
}

pub async fn document_set_metaword(db: &Database, url: STRHASH, word_name: STRHASH, property: STRHASH, occurances: &[u8]) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_metaword {e}");
    } else {
        let trx = trx.unwrap();
        // metaword -> word -> prop -> url
        // metawurl -> url -> prop -> word
        let subspace = Subspace::from((METAWORD_TABLE, word_name, property));
        let revsubspace = Subspace::from((METAWURL_TABLE, url, property));

        trx.set(&subspace.pack(&url), occurances);
        trx.set(&revsubspace.pack(&word_name), &pack(&word_name));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_metaword {e}");
        }
    }
}

pub async fn document_set_word(db: &Database, url: STRHASH, word_name: STRHASH, occurances: &[u8]) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_set_word {e}");
    } else {
        let trx = trx.unwrap();
        // word -> word -> url
        let subspace = Subspace::from((WORD_TABLE, word_name));
        let revsubspace = Subspace::from((WURL_TABLE, url));

        trx.set(&subspace.pack(&url), occurances);
        trx.set(&revsubspace.pack(&word_name), &pack(&word_name));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR document_set_word {e}");
        }
    }
}

pub async fn link_set_linkword(db: &Database, url: STRHASH, dest: STRHASH, word_name: STRHASH, occurances: &[u8]) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR link_set_linkword {e}");
    } else {
        let trx = trx.unwrap();
        // linkword -> word -> from -> to
        let subspace = Subspace::from((LINKWORD_TABLE, word_name, url));
        let revsubspace = Subspace::from((LINKWORD_TABLE, url, dest));

        trx.set(&subspace.pack(&dest), occurances);
        trx.set(&revsubspace.pack(&word_name), &pack(&word_name));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR link_set_linkword {e}");
        }
    }
}

pub async fn document_host(db: &Database, url: STRHASH) -> Option<STRHASH> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR document_host {e}");
        None
    } else {
        let trx = trx.unwrap();
        // website -> url -> host
        let subspace = Subspace::from((WEBSITE_TABLE, url));

        let v = trx.get(&subspace.pack(&WEBSITE_TABLE_HOST), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR document_host {e}");
            None
        } else {
            v.unwrap().map(|v| unpack::<STRHASH>(&v).expect("INVALID K TYPE document_last_parse_finish"))
        }
    }
}

pub async fn host_last_fetch(db: &Database, url: STRHASH) -> Option<chrono::DateTime<chrono::Utc>> {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR host_last_fetch {e}");
        None
    } else {
        let trx = trx.unwrap();
        // host -> url
        let subspace = Subspace::from(HOST_TABLE);

        let v = trx.get(&subspace.pack(&url), true).await;


        if let Err(e) = v {
            error!("DATABASE ERROR host_last_fetch {e}");
            None
        } else {
            v.unwrap().map(|v| chrono::Utc.timestamp_opt(unpack::<i64>(&v).expect("INVALID K TYPE host_last_fetch"), 0).unwrap())
        }
    }
}

pub async fn host_set_last_fetch(db: &Database, url: STRHASH, now: chrono::DateTime<chrono::Utc>) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR host_set_last_fetch {e}");
    } else {
        let trx = trx.unwrap();
        // host -> url
        let subspace = Subspace::from(HOST_TABLE);

        trx.set(&subspace.pack(&url), &pack(&now.timestamp()));

        if let Err(e) = trx.commit().await {
            error!("DATABASE ERROR host_set_last_fetch {e}");
        }
    }
}

pub async fn prevented_by_dehomogeniser(db: &Database, url: &str) -> (bool, String, String) {
    let trx = db.create_trx();
    if let Err(e) = trx {
        error!("DATABASE ERROR prevented_by_dehomogeniser {e}");
        (false, "".to_string(), "".to_string())
    } else {
        let trx = trx.unwrap();
        // revlink -> to -> *
        let subspace = Subspace::from(DEHOMOGENISE_TABLE);
        let range = RangeOption::from(&subspace);

        let kv = trx.get_range(&range, 1024, true).await;

        if let Err(e) = kv {
            error!("DATABASE ERROR prevented_by_dehomogeniser {e}");
            (false, "".to_string(), "".to_string())
        } else {
            let kv = kv.unwrap();

            for kv in kv.into_iter() {
                let (block, unless) = unpack::<(String, String)>(kv.value()).expect("INVALID V TYPE prevented_by_dehomogeniser");

                if url.contains(&block) && (unless.is_empty() || !url.contains(&unless)) {
                    return (true, block.to_string(), unless.to_string());
                }
            }

            (false, "".to_string(), "".to_string())
        }
    }
}