Data Mining in Java
Published:
In this blog, the mining data project contains seven steps: (1) Collecting Data (2) Text Pre-processing (3) Term Frequencies (4) Rugby and Term Co-Occurence (5) Data visualization (6) Sentiment Analysis basics (7) Geolocation and Interactive Maps
Intall tweepy which is a twitter API provided by twitter
pip install tweepy==3.3.0
In order to authorize our app to access Twitter on our behalf, we need to use OAuth interface:
import tweepy
from tweepy import OAuthHandler
consumer_key = 'YOUR-CONSUMER-KEY'
consumer_secret = 'YOUR-CONSUMER-SECRET'
access_token = 'YOUR-ACCESS-TOKEN'
access_secret = 'YOUR-ACCESS-SECRET'
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
Show the list of all your followers:
for friend in tweepy.Cursor(api.friends).items():
process_or_store(friend._json)
Show the list of all your tweets:
for tweet in tweepy.Cursor(api.user_timeline).items():
process_or_store(tweet._json)
Streaming: Keep the connection open and gather all that upcoming tweets:
# The function is to gather all the tweets with "#python" hashtag
from tweepy import Stream
from tweepy.streaming import StreamListener
class MyListener(StreamListener):
def on_data(self, data):
try:
with open('python.json', 'a') as f:
f.write(data)
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
return True
def on_error(self, status):
print(status)
return True
twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(track=['#python'])
Text pre-processing
with open('mytweets.json', 'r') as f:
for line in f:
tweet = json.loads(line)
tokens = preprocess(tweet['text'])
do_something_else(tokens)
Term frequencies:
import operator
import json
from collections import Counter
fname = 'mytweets.json'
with open(fname, 'r') as f:
count_all = Counter()
for line in f:
tweet = json.loads(line)
# Create a list with all the terms
terms_all = [term for term in preprocess(tweet['text'])]
# Update the counter
count_all.update(terms_all)
# Print the first 5 most frequent words
print(count_all.most_common(5))
Term Filters:
Remove stop words: remove words which are commonly appear in the tweets but doesn’t have very crucial meanings:(to,and.on)
from nltk.corpus import stopwords
import string
punctuation = list(string.punctuation)
stop = stopwords.words('english') + punctuation + ['rt', 'via']
# Count terms only once, equivalent to Document Frequency
terms_single = set(terms_all)
# Count hashtags only
terms_hash = [term for term in preprocess(tweet['text'])
if term.startswith('#')]
# Count terms only (no hashtags, no mentions)
terms_only = [term for term in preprocess(tweet['text'])
if term not in stop and
not term.startswith(('#', '@'))]
# mind the ((double brackets))
# startswith() takes a tuple (not a list) if
# we pass a list of inputs
Analyze the words which occurs together:
# Count terms only once, equivalent to Document Frequency
terms_single = set(terms_all)
# Count hashtags only
terms_hash = [term for term in preprocess(tweet['text'])
if term.startswith('#')]
# Count terms only (no hashtags, no mentions)
terms_only = [term for term in preprocess(tweet['text'])
if term not in stop and
not term.startswith(('#', '@'))]
# mind the ((double brackets))
# startswith() takes a tuple (not a list) if
# we pass a list of inputs
# About data visualization: Install Vincent:
sudo pip install vincent
select the most common 20 words to show:
import vincent
word_freq = count_terms_only.most_common(20)
labels, freq = zip(*word_freq)
data = {'data': freq, 'x': labels}
bar = vincent.Bar(data, iter_idx='x')
bar.to_json('term_freq.json')
Use html to show the bar:
<html>
<head>
<title>Vega Scaffold</title>
<script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script>
<script src="http://d3js.org/topojson.v1.min.js"></script>
<script src="http://d3js.org/d3.geo.projection.v0.min.js" charset="utf-8"></script>
<script src="http://trifacta.github.com/vega/vega.js"></script>
</head>
<body>
<div id="vis"></div>
</body>
<script type="text/javascript">
// parse a spec and create a visualization view
function parse(spec) {
vg.parse.spec(spec, function(chart) { chart({el:"#vis"}).update(); });
}
parse("term_freq.json");
</script>
</html>
Time series visualization:
Use pandas:
sudo pip install pandas
Use pandas to do the time series analysis
import pandas
import json
dates_ITAvWAL = []
# f is the file pointer to the JSON data set
for line in f:
tweet = json.loads(line)
# let's focus on hashtags only at the moment
terms_hash = [term for term in preprocess(tweet['text']) if term.startswith('#')]
# track when the hashtag is mentioned
if '#itavwal' in terms_hash:
dates_ITAvWAL.append(tweet['created_at'])
# a list of "1" to count the hashtags
ones = [1]*len(dates_ITAvWAL)
# the index of the series
idx = pandas.DatetimeIndex(dates_ITAvWAL)
# the actual series (at series of 1s for the moment)
ITAvWAL = pandas.Series(ones, index=idx)
# Resampling / bucketing
per_minute = ITAvWAL.resample('1Min', how='sum').fillna(0)
time_chart = vincent.Line(ITAvWAL)
time_chart.axis_titles(x='Time', y='Freq')
time_chart.to_json('time_chart.json')
# all the data together
match_data = dict(ITAvWAL=per_minute_i, SCOvIRE=per_minute_s, ENGvFRA=per_minute_e)
# we need a DataFrame, to accommodate multiple series
all_matches = pandas.DataFrame(data=match_data,
index=per_minute_i.index)
# Resampling as above
all_matches = all_matches.resample('1Min', how='sum').fillna(0)
# and now the plotting
time_chart = vincent.Line(all_matches[['ITAvWAL', 'SCOvIRE', 'ENGvFRA']])
time_chart.axis_titles(x='Time', y='Freq')
time_chart.legend(title='Matches')
time_chart.to_json('time_chart.json')
A simple way of sentiment analysis:
# n_docs is the total n. of tweets
p_t = {}
p_t_com = defaultdict(lambda : defaultdict(int))
for term, n in count_stop_single.items():
p_t[term] = n / n_docs
for t2 in com[term]:
p_t_com[term][t2] = com[term][t2] / n_docs
The original dataset of the sentiment:
positive_vocab = [
'good', 'nice', 'great', 'awesome', 'outstanding',
'fantastic', 'terrific', ':)', ':-)', 'like', 'love',
# shall we also include game-specific terms?
# 'triumph', 'triumphal', 'triumphant', 'victory', etc.
]
negative_vocab = [
'bad', 'terrible', 'crap', 'useless', 'hate', ':(', ':-(',
# 'defeat', etc.
]
Calculate the PMI:
pmi = defaultdict(lambda : defaultdict(int))
for t1 in p_t:
for t2 in com[t1]:
denom = p_t[t1] * p_t[t2]
pmi[t1][t2] = math.log2(p_t_com[t1][t2] / denom)
semantic_orientation = {}
for term, n in p_t.items():
positive_assoc = sum(pmi[term][tx] for tx in positive_vocab)
negative_assoc = sum(pmi[term][tx] for tx in negative_vocab)
semantic_orientation[term] = positive_assoc - negative_assoc
The Semantic Orientation of a term will have a positive (negative) value if the term is often
Interactive Map using :
<link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.3/leaflet.css" />
<script src="http://cdnjs.cloudflare.com/ajax/libs/leaflet/0.7.3/leaflet.js"></script>
<script src="http://code.jquery.com/jquery-2.1.0.min.js"></script>
<!-- this goes in the <head> -->
<style>
#map {
height: 600px;
}
</style>
<!-- this goes in the <body> -->
<div id="map"></div>
Create the map in with Leaflet
// Load the tile images from OpenStreetMap
var mytiles = L.tileLayer('http://{s}.tile.osm.org/{z}/{x}/{y}.png', {
attribution: '© <a href="http://osm.org/copyright">OpenStreetMap</a> contributors'
});
// Initialise an empty map
var map = L.map('map');
// Read the GeoJSON data with jQuery, and create a circleMarker element for each tweet
// Each tweet will be represented by a nice red dot
$.getJSON("./geo_data.json", function(data) {
var myStyle = {
radius: 2,
fillColor: "red",
color: "red",
weight: 1,
opacity: 1,
fillOpacity: 1
};
var geojson = L.geoJson(data, {
pointToLayer: function (feature, latlng) {
return L.circleMarker(latlng, myStyle);
}
});
geojson.addTo(map)
});
// Add the tiles to the map, and initialise the view in the middle of Europe
map.addLayer(mytiles).setView([50.5, 5.0], 5);