[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Librefm-commits] [1218] merge scripts from trunk
From: |
Clint Adams |
Subject: |
[Librefm-commits] [1218] merge scripts from trunk |
Date: |
Wed, 06 May 2009 00:25:03 +0000 |
Revision: 1218
http://svn.sv.gnu.org/viewvc/?view=rev&root=librefm&revision=1218
Author: clint
Date: 2009-05-06 00:25:02 +0000 (Wed, 06 May 2009)
Log Message:
-----------
merge scripts from trunk
Modified Paths:
--------------
branches/stable/scripts/import-jamendo.py
branches/stable/scripts/scrobbles-revisionism
branches/stable/scripts/users_dump.sh
Added Paths:
-----------
branches/stable/scripts/countrycodes.pl
branches/stable/scripts/gobble.py
branches/stable/scripts/most_recent.py
branches/stable/scripts/offline-submit.py
branches/stable/scripts/setstids
Removed Paths:
-------------
branches/stable/scripts/users.sql
Added: branches/stable/scripts/countrycodes.pl
===================================================================
--- branches/stable/scripts/countrycodes.pl (rev 0)
+++ branches/stable/scripts/countrycodes.pl 2009-05-06 00:25:02 UTC (rev
1218)
@@ -0,0 +1,54 @@
+#!/usr/bin/perl
+
+use CGI qw(escape);
+use Encode qw(_utf8_on);
+use LWP::Simple;
+use strict;
+
+my $data =
get('http://en.wikipedia.org/w/index.php?title=ISO_3166-1_alpha-2&action=raw');
+_utf8_on($data); # mess with Perl's internals... data is in UTF8, but we need
to tell Perl that!
+my @lines = split /\r?\n/, $data;
+
+my $longest_name = 0;
+my $longest_link = 0;
+
+print "DROP TABLE Countries;\n";
+print "CREATE TABLE Countries (country varchar(2) PRIMARY KEY, country_name
varchar(200), wikipedia_en varchar(120));\n";
+foreach (@lines)
+{
+ # toothpicks stood on end
+ if (/^ \s* \| \s* # Initial pipe symbol
+ id=\"([A-Z]{2})\" # ID attribute
+ \s* \| \s* # Another pipe symbol
+ <tt>..<\/tt> # Teletype font
+ \s* \|\| \s* # Double pipe symbol
+ \[\[ # Double opening brackets
+ ([^\]\|]+) # Link
+ (\|([^\]]+))? # Link text (optional)
+ \]\] # Double closing bracket
+ /xi)
+ {
+
+
+ my $code = $1;
+ my $link = escape($2);
+ my $name = $4 || $2;
+
+ $link =~ s/\%20/_/g;
+
+ $name =~ s/\'/\'\'/g;
+
+ $longest_link = length($link) if length($link) > $longest_link;
+ $longest_name = length($name) if length($name) > $longest_name;
+
+ print "INSERT INTO Countries VALUES ('$code', '$name',
'http://en.wikipedia.org/wiki/$link');\n";
+ }
+}
+
+$longest_link += length('http://en.wikipedia.org/wiki/');
+
+print "-- Longest name: $longest_name\n";
+print "-- Longest link: $longest_link\n";
+print "\n";
+print "-- This needs to be piped into the postgresql command line client.\n";
+print "-- e.g. ./countrycodes.pl | psql librefm\n";
Added: branches/stable/scripts/gobble.py
===================================================================
--- branches/stable/scripts/gobble.py (rev 0)
+++ branches/stable/scripts/gobble.py 2009-05-06 00:25:02 UTC (rev 1218)
@@ -0,0 +1,93 @@
+try:
+ import hashlib
+ md5hash = hashlib.md5
+except ImportError:
+ import md5
+ md5hash = md5.new
+import time
+from urllib import urlencode
+from urllib2 import urlopen
+
+
+class GobbleException(Exception):
+
+ pass
+
+
+class GobbleServer(object):
+
+ def __init__(self, server_name, username, password):
+ if server_name[:7] != "http://":
+ server_name = "http://%s" % (server_name,)
+ self.name = server_name
+ self.password = password
+ self.post_data = []
+ self.session_id = None
+ self.submit_url = None
+ self.username = username
+ self._handshake()
+
+
+ def _handshake(self):
+ timestamp = int(time.time())
+ token = (md5hash(md5hash(self.password).hexdigest()
+ + str(timestamp)).hexdigest())
+ auth_url = "%s/?hs=true&p=1.2&u=%s&t=%d&a=%s&c=imp" % (self.name,
+ self.username,
+ timestamp,
token)
+ response = urlopen(auth_url).read()
+ lines = response.split("\n")
+ if lines[0] != "OK":
+ raise GobbleException("Server returned: %s" % (response,))
+ self.session_id = lines[1]
+ self.submit_url = lines[3]
+
+ def submit(self):
+ if len(self.post_data) == 0:
+ return
+ i = 0
+ data = []
+ for track in self.post_data:
+ data += track.get_tuples(i)
+ i += 1
+ data += [('s', self.session_id)]
+ response = urlopen(self.submit_url, urlencode(data)).read()
+ if response != "OK\n":
+ raise GobbleException("Server returned: %s" % (response,))
+ self.post_data = []
+ time.sleep(1)
+
+ def add_track(self, gobble_track):
+ i = len(self.post_data)
+ if i > 49:
+ self.submit()
+ i = 0
+ self.post_data.append(gobble_track)
+
+
+class GobbleTrack(object):
+
+ def __init__(self, artist, track, timestamp, album=None, length=None,
+ tracknumber=None, mbid=None):
+ self.artist = artist
+ self.track = track
+ self.timestamp = timestamp
+ self.album = album
+ self.length = length
+ self.tracknumber = tracknumber
+ self.mbid = mbid
+
+ def get_tuples(self, i):
+ timestamp = str(int(time.mktime(self.timestamp.utctimetuple())))
+ data = []
+ data += [('a[%d]' % i, self.artist), ('t[%d]' % i, self.track),
+ ('i[%d]' % i, timestamp)]
+ if self.album is not None:
+ data.append(('b[%d]' % i, self.album))
+ if self.length is not None:
+ data.append(('l[%d]' % i, self.length))
+ if self.tracknumber is not None:
+ data.append(('n[%d]' % i, self.tracknumber))
+ if self.mbid is not None:
+ data.append(('m[%d]' % i, self.mbid))
+ return data
Modified: branches/stable/scripts/import-jamendo.py
===================================================================
--- branches/stable/scripts/import-jamendo.py 2009-05-06 00:15:05 UTC (rev
1217)
+++ branches/stable/scripts/import-jamendo.py 2009-05-06 00:25:02 UTC (rev
1218)
@@ -211,7 +211,7 @@
if
self.track_exists(artist["name"], album["name"], track["name"]):
try:
-
self.cursor.execute("UPDATE Track SET downloadurl = %s, streamurl = %s, mbid =
%s, license = %s, duration = %s, otherid = %s AND streamable = %s WHERE name =
%s AND artist = %s AND album = %s", (track["downloadurl"], track["streamurl"],
track["mbid"], track["license"], duration, otherid, streamable, track["name"],
artist["name"], album["name"]))
+
self.cursor.execute("UPDATE Track SET downloadurl = %s, streamurl = %s, mbid =
%s, license = %s, duration = %s, otherid = %s, streamable = %s WHERE name = %s
AND artist = %s AND album = %s", (track["downloadurl"], track["streamurl"],
track["mbid"], track["license"], duration, otherid, streamable, track["name"],
artist["name"], album["name"]))
self.conn.commit()
except Exception, e:
self.conn.rollback()
Added: branches/stable/scripts/most_recent.py
===================================================================
--- branches/stable/scripts/most_recent.py (rev 0)
+++ branches/stable/scripts/most_recent.py 2009-05-06 00:25:02 UTC (rev
1218)
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+from datetime import datetime
+from optparse import OptionParser
+from urllib2 import urlopen
+
+from BeautifulSoup import BeautifulSoup
+
+
+if __name__ == '__main__':
+ usage = "%prog <USERNAME> <WEB SERVER>"
+ parser = OptionParser(usage=usage)
+ opts,args = parser.parse_args()
+ if len(args) != 2:
+ parser.error("All arguments are required.")
+
+ username,server = args
+ if server[:7] != "http://":
+ server = "http://%s" % (server,)
+ soup = BeautifulSoup(urlopen('%s/user/%s/recent-tracks'
+ % (server, username)))
+ gobbles_list = soup.find('ul', {'class': 'gobbles'})
+ timestamp = gobbles_list.find('li')['about'].split('#')[1].split('.')[0]
+ print datetime.fromtimestamp(float(timestamp))
Added: branches/stable/scripts/offline-submit.py
===================================================================
--- branches/stable/scripts/offline-submit.py (rev 0)
+++ branches/stable/scripts/offline-submit.py 2009-05-06 00:25:02 UTC (rev
1218)
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+import datetime
+import getpass
+from optparse import OptionParser
+import subprocess
+
+import mutagen
+from mutagen import easyid3
+
+from gobble import GobbleServer, GobbleTrack
+
+
+def _parse_date(string):
+ process = subprocess.Popen(['date', '-d %s' % (string,), '+%s'],
+ stdout=subprocess.PIPE)
+ string = process.communicate()[0].strip()
+ return datetime.datetime.utcfromtimestamp(float(string))
+
+
+if __name__ == '__main__':
+ usage = "%prog <USERNAME> <SERVER> <START TIME> <MEDIA FILES>"
+ parser = OptionParser(usage=usage)
+ opts,args = parser.parse_args()
+ if len(args) < 4:
+ parser.error("All arguments are required.")
+
+ username,server,start_string = args[:3]
+ password = getpass.getpass()
+ tracks = args[3:]
+ server = GobbleServer(server, username, password)
+
+ dt = _parse_date(start_string)
+
+ for track in tracks:
+ f = mutagen.File(track)
+ if f is None:
+ raise Exception("%s caused problems." % (track,))
+ if isinstance(f, mutagen.mp3.MP3):
+ f = mutagen.mp3.MP3(track, ID3=easyid3.EasyID3)
+ title = f['title'][0]
+ artist = f['artist'][0]
+ length = f.info.length
+ album = f['album'][0]
+ tracknumber = f['tracknumber'][0]
+ t = GobbleTrack(artist, title, dt, album=album, length=length,
+ tracknumber=tracknumber)
+ server.add_track(t)
+ dt += datetime.timedelta(seconds=length)
+ server.submit()
Modified: branches/stable/scripts/scrobbles-revisionism
===================================================================
--- branches/stable/scripts/scrobbles-revisionism 2009-05-06 00:15:05 UTC
(rev 1217)
+++ branches/stable/scripts/scrobbles-revisionism 2009-05-06 00:25:02 UTC
(rev 1218)
@@ -41,6 +41,9 @@
if ">" in oldname:
newname = newname.replace(">",">")
+ if " (PREVIEW: buy it at www.magnatune.com)" in oldname:
+ newname = newname.replace(" (PREVIEW: buy it at
www.magnatune.com)","")
+
if oldname <> newname:
print "%s >>>INTO>>> %s" % (oldname, newname)
try:
Added: branches/stable/scripts/setstids
===================================================================
--- branches/stable/scripts/setstids (rev 0)
+++ branches/stable/scripts/setstids 2009-05-06 00:25:02 UTC (rev 1218)
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import sys;
+import adodb;
+
+class CleanUp:
+
+ def __init__(self, dbname, username):
+ self.conn = adodb.NewADOConnection('postgres')
+ self.conn.Connect('user=%s dbname=%s' % (username, dbname))
+
+ def doit(self):
+
+ self.cursor = self.conn.Execute('SELECT DISTINCT artist, album,
track, mbid from Scrobbles WHERE stid is NULL');
+
+ while not self.cursor.EOF:
+ row = self.cursor.GetRowAssoc(0)
+ artist = row['artist']
+ album = row['album']
+ track = row['track']
+ mbid = row['mbid']
+ stid = self.lookupStid(artist,album,track,mbid)
+ if not stid:
+ tid = self.lookupTrack(artist, album, track,
mbid)
+ if tid:
+ print "Adding stid for %s/%s/%s/%s/%s"
% (tid,artist,album,track,mbid)
+ self.acursor =
self.conn.Execute("INSERT INTO Scrobble_Track (artist,album,name,mbid,track)
VALUES(lower(%(artist)s),lower(%(album)s),lower(%(name)s),lower(%(mbid)s),%(track)s)",
{'artist': artist, 'album': album, 'name': track, 'mbid': mbid, 'track': tid})
+ else:
+ print "Adding tid for %s/%s/%s/%s" %
(artist,album,track,mbid)
+ self.acursor =
self.conn.Execute("INSERT INTO Track (artist,album,name,mbid)
VALUES(%(artist)s,%(album)s,%(name)s,%(mbid)s)", {'artist': artist, 'album':
album, 'name': track, 'mbid': mbid})
+ else:
+ print "Setting %d to match %s/%s/%s/%s" %
(stid,artist,album,track,mbid)
+ self.scursor = self.conn.Execute("UPDATE
Scrobbles SET stid = %%(stid)s WHERE track = %%(track)s AND artist =
%%(artist)s AND %s AND %s AND stid IS NULL" % (('lower(album) =
lower(%(album)s)' if album else 'album IS NULL'), ('lower(mbid) =
lower(%(mbid)s)' if mbid else 'mbid IS NULL')), {'stid': stid, 'track': track,
'artist': artist, 'album': album, 'mbid': mbid})
+ self.cursor.MoveNext()
+
+ def lookupStid(self, artist, album, track, mbid):
+ self.lucursor = self.conn.Execute('SELECT * from Scrobble_Track
WHERE lower(artist) = lower(%%(artist)s) AND %s AND lower(name) =
lower(%%(track)s) AND %s' % (('lower(album) = lower(%(album)s)' if album else
'album IS NULL'), ('lower(mbid) = lower(%(mbid)s)' if mbid else 'mbid IS
NULL')), {'artist': artist, 'album': album, 'track': track, 'mbid': mbid});
+ if not self.lucursor.EOF:
+ lrow = self.lucursor.GetRowAssoc(0)
+ return lrow['id']
+ else:
+ return None
+
+ def lookupTrack(self, artist, album, track, mbid):
+ self.tucursor = self.conn.Execute('SELECT * from Track WHERE
lower(artist) = lower(%%(artist)s) AND %s AND lower(name) = lower(%%(track)s)
AND %s' % (('lower(album) = lower(%(album)s)' if album else 'album IS NULL'),
('lower(mbid) = lower(%(mbid)s)' if mbid else 'mbid IS NULL')), {'artist':
artist, 'album': album, 'track': track, 'mbid': mbid});
+ if not self.tucursor.EOF:
+ trow = self.tucursor.GetRowAssoc(0)
+ return trow['id']
+ else:
+ return None
+
+ def close(self):
+ self.cursor.Close()
+ self.conn.Close()
+
+if __name__ == "__main__":
+
+ if len(sys.argv) != 3:
+ print "Usage: setstids <pgsql dbname> <pgsql username>"
+ sys.exit(1)
+
+ cleaner = CleanUp(sys.argv[1], sys.argv[2])
+ cleaner.doit()
+ cleaner.close()
Deleted: branches/stable/scripts/users.sql
===================================================================
--- branches/stable/scripts/users.sql 2009-05-06 00:15:05 UTC (rev 1217)
+++ branches/stable/scripts/users.sql 2009-05-06 00:25:02 UTC (rev 1218)
@@ -1 +0,0 @@
-SELECT username FROM `Users`
Modified: branches/stable/scripts/users_dump.sh
===================================================================
--- branches/stable/scripts/users_dump.sh 2009-05-06 00:15:05 UTC (rev
1217)
+++ branches/stable/scripts/users_dump.sh 2009-05-06 00:25:02 UTC (rev
1218)
@@ -2,6 +2,14 @@
cd /home/librefm/turtle/data/wip || exit 1
+if [ -x /usr/bin/lockfile-create ]; then
+ lockfile-create /var/lock/gnukebox-userdump.lock
+ lockfile-touch /var/lock/gnukebox-userdump.lock &
+ LOCKER="$!"
+else
+ echo >&2 "Running without locking."
+fi
+
#we rely on lack of whitespace here
LIST=$(echo "SELECT DISTINCT username FROM Users;" | psql -q -t)
@@ -11,5 +19,10 @@
echo 'COPY (SELECT * FROM Scrobbles where username='"'$I'"') TO STDOUT
WITH CSV HEADER;' | psql -q >> $I.text.utf8
- mv -- $I /home/librefm/turtle/data/
+ mv -- $I.text.utf8 /home/librefm/turtle/data/
done
+
+if [ -x /usr/bin/lockfile-create ]; then
+ kill "${LOCKER}"
+ lockfile-remove /var/lock/gnukebox-userdump.lock
+fi
[Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Librefm-commits] [1218] merge scripts from trunk,
Clint Adams <=