AustrianToast/HDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove golang stuff

Browse Source
This commit is contained in:
AustrianToast
2024-11-01 13:45:17 +01:00
parent dc0d30fa4e
commit ee9f85d158
5 changed files with 1 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Hentai_create.sql
View File

@ -1,20 +0,0 @@
CREATE TABLE artist (
id integer NOT NULL CONSTRAINT artist_pk PRIMARY KEY AUTOINCREMENT,
name text NOT NULL
);
CREATE TABLE links (
id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT,
url text NOT NULL,
website_id integer NOT NULL,
artist_id integer NOT NULL,
CONSTRAINT links_artist FOREIGN KEY (artist_id)
REFERENCES artist (id),
CONSTRAINT links_website FOREIGN KEY (website_id)
REFERENCES website (id)
);
CREATE TABLE website (
id integer NOT NULL CONSTRAINT website_pk PRIMARY KEY AUTOINCREMENT,
name text NOT NULL
);

2
gallery-dl/Rule34.links
View File

@ -1932,4 +1932,4 @@ https://rule34.xxx/index.php?page=post&s=list&tags=zzpai
https://rule34.xxx/index.php?page=post&s=list&tags=zzzhodazzz https://rule34.xxx/index.php?page=post&s=list&tags=zzzhodazzz
https://rule34.xxx/index.php?page=post&s=list&tags=%E3%81%91%E3%82%93%E3%82%8D%E3%81%8F%E3%83%91%E3%83%BC%E3%82%AF https://rule34.xxx/index.php?page=post&s=list&tags=%E3%81%91%E3%82%93%E3%82%8D%E3%81%8F%E3%83%91%E3%83%BC%E3%82%AF
https://rule34.xxx/index.php?page=post&s=list&tags=%E3%81%A8%E3%82%89tora https://rule34.xxx/index.php?page=post&s=list&tags=%E3%81%A8%E3%82%89tora
https://rule34.xxx/index.php?page=post&s=list&tags=%e6%9f%8a%e5%a3%ac https://rule34.xxx/index.php?page=post&s=list&tags=%E6%9F%8A%E5%A3%AC

14
go.mod
View File

@ -1,14 +0,0 @@
module hentai
go 1.23.1
require github.com/ncruces/go-sqlite3 v0.18.3
require golang.org/x/sync v0.8.0 // indirect
require (
github.com/ncruces/julianday v1.0.0 // indirect
github.com/otiai10/copy v1.14.0
github.com/tetratelabs/wazero v1.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
)

12
go.sum
View File

@ -1,12 +0,0 @@
github.com/ncruces/go-sqlite3 v0.18.3 h1:tyMa75uh7LcINcfo0WrzOvcTkfz8Hqu0TEPX+KVyes4=
github.com/ncruces/go-sqlite3 v0.18.3/go.mod h1:HAwOtA+cyEX3iN6YmkpQwfT4vMMgCB7rQRFUdOgEFik=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

219
main.go
View File

@ -1,219 +0,0 @@
package main
import (
"bufio"
"fmt"
"log"
"net/url"
"os"
"strings"
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
"github.com/otiai10/copy"
)
const memory = ":memory:"
var db *sqlite3.Conn
func initDb() error {
var err error
var sqlstmt string
// Not really necessary because I am currently saving the db in memory
sqlstmt = `
DROP TABLE IF EXISTS artists;
DROP TABLE IF EXISTS websites;
DROP TABLE IF EXISTS links;
`
err = db.Exec(sqlstmt)
if err != nil {
log.Fatal(err)
}
sqlstmt = `
CREATE TABLE IF NOT EXISTS artists (
id integer NOT NULL CONSTRAINT artists_pk PRIMARY KEY AUTOINCREMENT,
name text NOT NULL
);
CREATE TABLE IF NOT EXISTS websites (
id integer NOT NULL CONSTRAINT websites_pk PRIMARY KEY AUTOINCREMENT,
name text NOT NULL
);
CREATE TABLE IF NOT EXISTS links (
id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT,
url text NOT NULL,
websites_id integer NOT NULL,
artists_id integer NOT NULL,
CONSTRAINT links_artists FOREIGN KEY (artists_id)
REFERENCES artists (id),
CONSTRAINT links_websites FOREIGN KEY (websites_id)
REFERENCES websites (id)
);
`
err = db.Exec(sqlstmt)
if err != nil {
return err
}
sqlstmt = `
SELECT name
FROM sqlite_schema
WHERE type ='table' AND name NOT LIKE 'sqlite_%';
`
stmt, _, err := db.Prepare(sqlstmt)
if err != nil {
return err
}
defer stmt.Close()
for stmt.Step() {
log.Println("created table:", stmt.ColumnText(0))
}
if err := stmt.Err(); err != nil {
return err
}
err = stmt.Close()
if err != nil {
return err
}
return nil
}
func getArtistFromUrl(Url string) string {
if strings.Contains(Url, "rule34.xxx") {
unescapedUrl, err := url.QueryUnescape(Url)
if err != nil {
log.Fatal(err)
}
return unescapedUrl[51:]
}
if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") {
return strings.Split(Url, "/")[5]
}
return ""
}
func trimUrl(url string) string {
// we can look for specific strings like encoded chars and then remove them
disallowedStrings := []string{
"%20", "%21", "%22", "%23", "%24",
"%26", "%27", "%28", "%29", "%2A",
"%2C", "%2E", "%2F", "%3B", "%3C",
"%3E", "%3F", "%5B", "%5C", "%5D",
"%5E", "%60", "%7B", "%7C", "%7D",
"%7E", "artist",
}
for _, disallowedString := range disallowedStrings {
url = strings.ReplaceAll(url, disallowedString, "")
}
return strings.TrimRight(url, "-_")
}
func convert(fileLine string) error {
log.Println("starting conversion on:", fileLine)
oldPath := fmt.Sprintf("Artists/%s", getArtistFromUrl(fileLine))
fileLine = trimUrl(fileLine)
log.Println("trimmed into:", fileLine)
if strings.Contains(fileLine, "%") {
return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made")
}
artist := getArtistFromUrl(fileLine)
if strings.Compare(artist, "") == 0 {
return fmt.Errorf("%s", "Trying to get the artist name from the url failed")
}
log.Println("artist name:", artist)
newPath := fmt.Sprintf("Artists/%s", artist)
if len(artist) <= 1 {
return fmt.Errorf("%s", "Trying to remove unwanted stuff from artist name has caused it to become too small")
}
if strings.Compare(oldPath, newPath) == 0 {
return nil
}
log.Printf("oldPath: %s\tnewPath: %s\n", oldPath, newPath)
err := copy.Copy(oldPath, newPath)
if err != nil {
return err
}
os.Rename(oldPath, fmt.Sprint("Artist/__", artist))
return nil
}
func parseLinks(filename string) error {
file, err := os.Open(filename)
if err != nil {
return err
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
var line string
line = scanner.Text()
if strings.HasPrefix(line, "#") {
continue
}
if !strings.Contains(line, "#") {
err = convert(line)
if err != nil {
log.Printf("convert: %s\n", err)
}
fmt.Println()
continue
}
/*
split_url := strings.Split(line, "#")
url := strings.TrimSpace(split_url[0])
artist := strings.TrimSpace(split_url[1])
log.Printf("url: %s\tartist: %s\n", url, artist)
*/
}
err = file.Close()
if err != nil {
return err
}
return nil
}
func main() {
var err error
db, err = sqlite3.Open(memory)
if err != nil {
log.Fatal(err)
}
err = initDb()
if err != nil {
log.Fatal(err)
}
err = parseLinks("gallery-dl/Rule34.links")
if err != nil {
log.Fatal(err)
}
err = db.Close()
if err != nil {
log.Fatal(err)
}
}