HDB/main.go

package main

import (
	"bufio"
	"fmt"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/ncruces/go-sqlite3"
	_ "github.com/ncruces/go-sqlite3/embed"
	// "github.com/otiai10/copy"
)

const memory = ":memory:"
var db *sqlite3.Conn

func initDb() error {
	var err error
	/* Not necessary because I am currently saving the db in memory
	sqlstmt := `
		DROP TABLE IF EXISTS artists;
		DROP TABLE IF EXISTS websites;
		DROP TABLE IF EXISTS links;
	`
	err = db.Exec(sqlstmt)
	if err != nil {
		log.Fatal(err)
	}
	*/

	sqlstmt := `
	CREATE TABLE IF NOT EXISTS artists (
		id integer NOT NULL CONSTRAINT artists_pk PRIMARY KEY AUTOINCREMENT,
		name text NOT NULL
	);
	CREATE TABLE IF NOT EXISTS websites (
		id integer NOT NULL CONSTRAINT websites_pk PRIMARY KEY AUTOINCREMENT,
		name text NOT NULL
	);
	CREATE TABLE IF NOT EXISTS links (
		id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT,
		url text NOT NULL,
		websites_id integer NOT NULL,
		artists_id integer NOT NULL,
		CONSTRAINT links_artists FOREIGN KEY (artists_id)
		REFERENCES artists (id),
		CONSTRAINT links_websites FOREIGN KEY (websites_id)
		REFERENCES websites (id)
	);
	`
	err = db.Exec(sqlstmt)
	if err != nil {
		return err
	}

	sqlstmt = `
		SELECT name
		FROM sqlite_schema
		WHERE type ='table' AND name NOT LIKE 'sqlite_%';
	`

	stmt, _, err := db.Prepare(sqlstmt)
	if err != nil {
		return err
	}
	defer stmt.Close()

	for stmt.Step() {
		log.Println("created table:", stmt.ColumnText(0))
	}
	if err := stmt.Err(); err != nil {
		return err
	}

	err = stmt.Close()
	if err != nil {
		return err
	}

	return nil
}

func getArtistFromUrl(Url string) string {
	if strings.Contains(Url, "rule34.xxx") {
		unescapedUrl, err := url.QueryUnescape(Url)
		if err != nil {
			log.Fatal(err)
		}
		return unescapedUrl[51:]
	}
	if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") {
		return strings.Split(Url, "/")[5]
	}

	return ""
}

func convert(fileLine string) error {
	log.Println(fileLine)
	if strings.Contains(fileLine, "%") {
		// we can look for specific encoded chars and the remove them and continue instead of returning an error
		return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made")
	}
	artist := getArtistFromUrl(fileLine)
	log.Println("artist:", artist)
	//path := fmt.Sprintf("Artists/%s", artist)
	//log.Println("path:", path)

	if strings.ContainsAny(artist, "!\"#$&'()*,;<>?[\\]^`{|}~") {
		return fmt.Errorf("%s", "artistName contains disallowed characters and a direct conversion cannot be reasonably made")
	}

	return nil
}

func parseLinks(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		var line string
		line = scanner.Text()

		if strings.HasPrefix(line, "#") {
			continue
		}
		if !strings.Contains(line, "#") {
			err = convert(line)
			if err != nil {
				log.Printf("error: %s\n", err)
				continue
			}
			continue
		}
		/*
		split_url := strings.Split(line, "#")
		url := strings.TrimSpace(split_url[0])
		artist := strings.TrimSpace(split_url[1])

		log.Printf("url: %s\tartist: %s\n", url, artist)
		*/
	}

	err = file.Close()
	if err != nil {
		return err
	}

	return nil
}

func main() {
	var err error
	db, err = sqlite3.Open(memory)
	if err != nil {
		log.Fatal(err)
	}

	err = initDb()
	if err != nil {
		log.Fatal(err)
	}

	err = parseLinks("gallery-dl/Rule34.links")
	if err != nil {
		log.Fatal(err)
	}

	err = db.Close()
	if err != nil {
		log.Fatal(err)
	}
}