AustrianToast/HDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress on convert()

Browse Source
This commit is contained in:
AustrianToast
2024-09-27 16:33:16 +02:00
parent 7d01019077
commit 7011c75530
3 changed files with 48 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
main.go
View File

@ -2,6 +2,7 @@ package main
import (
"bufio"
"fmt"
"log"
"net/url"
"os"
@ -9,6 +10,7 @@ import (
"github.com/ncruces/go-sqlite3"
_ "github.com/ncruces/go-sqlite3/embed"
// "github.com/otiai10/copy"
)
const memory = ":memory:"
@ -88,16 +90,31 @@ func getArtistFromUrl(Url string) string {
}
return unescapedUrl[51:]
}
if strings.Contains(Url, "kemono.su") {
return "Kemono"
}
if strings.Contains(Url, "coomer.su") {
return "Coomer"
if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") {
return strings.Split(Url, "/")[5]
}
return ""
}
func convert(fileLine string) error {
log.Println(fileLine)
if strings.Contains(fileLine, "%") {
// we can look for specific encoded chars and the remove them and continue instead of returning an error
return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made")
}
artist := getArtistFromUrl(fileLine)
log.Println("artist:", artist)
//path := fmt.Sprintf("Artists/%s", artist)
//log.Println("path:", path)
if strings.ContainsAny(artist, "!\"#$&'()*,;<>?[\\]^`{|}~") {
return fmt.Errorf("%s", "artistName contains disallowed characters and a direct conversion cannot be reasonably made")
}
return nil
}
func parseLinks(filename string) error {
file, err := os.Open(filename)
if err != nil {
@ -106,14 +123,27 @@ func parseLinks(filename string) error {
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
var line string
line = scanner.Text()
if strings.HasPrefix(line, "#") {
continue
}
if !strings.Contains(line, "DBPath") {
if !strings.Contains(line, "#") {
err = convert(line)
if err != nil {
log.Printf("error: %s\n", err)
continue
}
continue
}
/*
split_url := strings.Split(line, "#")
url := strings.TrimSpace(split_url[0])
artist := strings.TrimSpace(split_url[1])
log.Printf("url: %s\tartist: %s\n", url, artist)
*/
}
err = file.Close()
@ -136,8 +166,10 @@ func main() {
log.Fatal(err)
}
artist := getArtistFromUrl("https://rule34.xxx/index.php?page=post&s=list&tags=%27eadole")
log.Println("artist from url: ", artist)
err = parseLinks("gallery-dl/Rule34.links")
if err != nil {
log.Fatal(err)
}
err = db.Close()
if err != nil {