package main import ( "bufio" "fmt" "log" "net/url" "os" "strings" "github.com/ncruces/go-sqlite3" _ "github.com/ncruces/go-sqlite3/embed" // "github.com/otiai10/copy" ) const memory = ":memory:" var db *sqlite3.Conn func initDb() error { var err error /* Not necessary because I am currently saving the db in memory sqlstmt := ` DROP TABLE IF EXISTS artists; DROP TABLE IF EXISTS websites; DROP TABLE IF EXISTS links; ` err = db.Exec(sqlstmt) if err != nil { log.Fatal(err) } */ sqlstmt := ` CREATE TABLE IF NOT EXISTS artists ( id integer NOT NULL CONSTRAINT artists_pk PRIMARY KEY AUTOINCREMENT, name text NOT NULL ); CREATE TABLE IF NOT EXISTS websites ( id integer NOT NULL CONSTRAINT websites_pk PRIMARY KEY AUTOINCREMENT, name text NOT NULL ); CREATE TABLE IF NOT EXISTS links ( id integer NOT NULL CONSTRAINT links_pk PRIMARY KEY AUTOINCREMENT, url text NOT NULL, websites_id integer NOT NULL, artists_id integer NOT NULL, CONSTRAINT links_artists FOREIGN KEY (artists_id) REFERENCES artists (id), CONSTRAINT links_websites FOREIGN KEY (websites_id) REFERENCES websites (id) ); ` err = db.Exec(sqlstmt) if err != nil { return err } sqlstmt = ` SELECT name FROM sqlite_schema WHERE type ='table' AND name NOT LIKE 'sqlite_%'; ` stmt, _, err := db.Prepare(sqlstmt) if err != nil { return err } defer stmt.Close() for stmt.Step() { log.Println("created table:", stmt.ColumnText(0)) } if err := stmt.Err(); err != nil { return err } err = stmt.Close() if err != nil { return err } return nil } func getArtistFromUrl(Url string) string { if strings.Contains(Url, "rule34.xxx") { unescapedUrl, err := url.QueryUnescape(Url) if err != nil { log.Fatal(err) } return unescapedUrl[51:] } if strings.Contains(Url, "kemono.su") || strings.Contains(Url, "coomer.su") { return strings.Split(Url, "/")[5] } return "" } func convert(fileLine string) error { log.Println(fileLine) if strings.Contains(fileLine, "%") { // we can look for specific encoded chars and the remove them and continue instead of returning an error return fmt.Errorf("%s", "URL contains Encoded Unicode and a direct conversion cannot be reasonably made") } artist := getArtistFromUrl(fileLine) log.Println("artist:", artist) //path := fmt.Sprintf("Artists/%s", artist) //log.Println("path:", path) if strings.ContainsAny(artist, "!\"#$&'()*,;<>?[\\]^`{|}~") { return fmt.Errorf("%s", "artistName contains disallowed characters and a direct conversion cannot be reasonably made") } return nil } func parseLinks(filename string) error { file, err := os.Open(filename) if err != nil { return err } scanner := bufio.NewScanner(file) for scanner.Scan() { var line string line = scanner.Text() if strings.HasPrefix(line, "#") { continue } if !strings.Contains(line, "#") { err = convert(line) if err != nil { log.Printf("error: %s\n", err) continue } continue } /* split_url := strings.Split(line, "#") url := strings.TrimSpace(split_url[0]) artist := strings.TrimSpace(split_url[1]) log.Printf("url: %s\tartist: %s\n", url, artist) */ } err = file.Close() if err != nil { return err } return nil } func main() { var err error db, err = sqlite3.Open(memory) if err != nil { log.Fatal(err) } err = initDb() if err != nil { log.Fatal(err) } err = parseLinks("gallery-dl/Rule34.links") if err != nil { log.Fatal(err) } err = db.Close() if err != nil { log.Fatal(err) } }