HDB/main.go

package main

import (
	"bufio"
	"fmt"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/ncruces/go-sqlite3"
	_ "github.com/ncruces/go-sqlite3/embed"
	"github.com/otiai10/copy"
)

const memory = ":memory:"
var db *sqlite3.Conn

func initDb() error {
	var err error
	var sqlstmt string

	// Not really necessary because I am currently saving the db in memory
	sqlstmt = `
		DROP TABLE IF EXISTS artists;
		DROP TABLE IF EXISTS websites;
		DROP TABLE IF EXISTS links;
	`
	err = db.Exec(sqlstmt)
	if err != nil {
		log.Fatal(err)
	}

	sqlstmt = `
	CREATE TABLE IF NOT EXISTS artists (
		id integer NOT NULL CONSTRAINT artists_pk PRIMARY KEY AUTOINCREMENT,
		name text NOT NULL
	);
	CREATE TABLE IF NOT EXISTS websites (
		id integer NOT NULL CONSTRAINT websites_pk PRIMARY KEY AUTOINCREMENT,
		name text NOT NULL
	);
	CREATE TABLE IF NOT EXISTS links (
		id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT,
		url text NOT NULL,
		websites_id integer NOT NULL,
		artists_id integer NOT NULL,
		CONSTRAINT links_artists FOREIGN KEY (artists_id)
		REFERENCES artists (id),
		CONSTRAINT links_websites FOREIGN KEY (websites_id)
		REFERENCES websites (id)
	);
	`
	err = db.Exec(sqlstmt)
	if err != nil {
		return err
	}

	sqlstmt = `
		SELECT name
		FROM sqlite_schema
		WHERE type ='table' AND name NOT LIKE 'sqlite_%';
	`

	stmt, _, err := db.Prepare(sqlstmt)
	if err != nil {
		return err
	}
	defer stmt.Close()

	for stmt.Step() {
		log.Println("created table:", stmt.ColumnText(0))
	}
	if err := stmt.Err(); err != nil {
		return err
	}

	err = stmt.Close()
	if err != nil {
		return err
	}

	return nil
}

func getArtistFromUrl(Url string) string {
	if strings.Contains(Url, "rule34.xxx") {
		unescapedUrl, err := url.QueryUnescape(Url)
		if err != nil {
			log.Fatal(err)
		}
		return unescapedUrl[51:]
	}
	if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") {
		return strings.Split(Url, "/")[5]
	}

	return ""
}

func trimUrl(url string) string {
	// we can look for specific strings like encoded chars and then remove them
	disallowedStrings := []string{
		"%20", "%21", "%22", "%23", "%24",
		"%26", "%27", "%28", "%29", "%2A",
		"%2C", "%2E", "%2F", "%3B", "%3C",
		"%3E", "%3F", "%5B", "%5C", "%5D",
		"%5E", "%60", "%7B", "%7C", "%7D",
		"%7E", "artist",
	}

	for _, disallowedString := range disallowedStrings {
		url = strings.ReplaceAll(url, disallowedString, "")
	}
	return strings.TrimRight(url, "-_")
}

func convert(fileLine string) error {
	log.Println("starting conversion on:", fileLine)

	oldPath := fmt.Sprintf("Artists/%s", getArtistFromUrl(fileLine))

	fileLine = trimUrl(fileLine)

	log.Println("trimmed into:", fileLine)

	if strings.Contains(fileLine, "%") {
		return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made")
	}

	artist := getArtistFromUrl(fileLine)
	if strings.Compare(artist, "") == 0 {
		return fmt.Errorf("%s", "Trying to get the artist name from the url failed")
	}

	log.Println("artist name:", artist)
	newPath := fmt.Sprintf("Artists/%s", artist)

	if len(artist) <= 1 {
		return fmt.Errorf("%s", "Trying to remove unwanted stuff from artist name has caused it to become too small")
	}

	if strings.Compare(oldPath, newPath) == 0 {
		return nil
	}

	log.Printf("oldPath: %s\tnewPath: %s\n", oldPath, newPath)

	err := copy.Copy(oldPath, newPath)
	if err != nil {
		return err
	}

	os.Rename(oldPath, fmt.Sprint("Artist/__", artist))

	return nil
}

func parseLinks(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		var line string
		line = scanner.Text()

		if strings.HasPrefix(line, "#") {
			continue
		}
		if !strings.Contains(line, "#") {
			err = convert(line)
			if err != nil {
				log.Printf("convert: %s\n", err)
			}
			fmt.Println()
			continue
		}
		/*
		split_url := strings.Split(line, "#")
		url := strings.TrimSpace(split_url[0])
		artist := strings.TrimSpace(split_url[1])

		log.Printf("url: %s\tartist: %s\n", url, artist)
		*/
	}

	err = file.Close()
	if err != nil {
		return err
	}

	return nil
}

func main() {
	var err error
	db, err = sqlite3.Open(memory)
	if err != nil {
		log.Fatal(err)
	}

	err = initDb()
	if err != nil {
		log.Fatal(err)
	}

	err = parseLinks("gallery-dl/Rule34.links")
	if err != nil {
		log.Fatal(err)
	}

	err = db.Close()
	if err != nil {
		log.Fatal(err)
	}
}