Streaming Twitter NX Tweet Graph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tweepy\n",
    "import json\n",
    "import os\n",
    "import datetime, time\n",
    "import re\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "home_dir = \"/Users/christopherallison/.virtualenvs/py_twi/streaming_results\"\n",
    "save_dir = \"/Users/christopherallison/Documents/Coding/Gephi/twitter_stream\"\n",
    "\n",
    "# Enter your search queries here\n",
    "search_queries = \"harper trudeau\".split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for data prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "first_cap_re = re.compile('(.)([A-Z][a-z]+)')\n",
    "all_cap_re = re.compile('([a-z0-9])([A-Z])')\n",
    "\n",
    "def convert(name):\n",
    "    # Convert text to camel_case\n",
    "    s1 = first_cap_re.sub(r'\\1_\\2', name)\n",
    "    return all_cap_re.sub(r'\\1_\\2', s1).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_user(tweet_id, tweet_dict):\n",
    "    # Find tweet author based on tweet status ID\n",
    "    try:\n",
    "        x = tweet_dict[tweet_id]['user_screen_name']\n",
    "    except KeyError:\n",
    "        x = None # User is out of scope\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def twitter_data_to_graph(search_query):\n",
    "    \n",
    "    for sq in search_query:\n",
    "        \n",
    "        tweets = {}\n",
    "        \n",
    "        # Open source files for each query\n",
    "        \n",
    "        with open(os.path.join(home_dir, '{}_stream.json'.format(sq)), 'r') as f:\n",
    "        \n",
    "            for data in f:\n",
    "        \n",
    "                result = json.loads(data)\n",
    "\n",
    "                id_str = result['id_str']\n",
    "\n",
    "                tweets[id_str] = {\n",
    "                    'id_str': id_str,\n",
    "                    'date': str(result['created_at']),\n",
    "                    'text': result['text'],\n",
    "                    'retweet_count': result['retweet_count'],\n",
    "                    'favorite_count': result['favorite_count'],\n",
    "                    'reply_to': result['in_reply_to_screen_name'],\n",
    "                    'coordinates': result['coordinates'],\n",
    "                    'reply_to_tweet': result['in_reply_to_status_id'],\n",
    "                    'user_screen_name': result['user']['screen_name'],\n",
    "                    'quoted_status': result['is_quote_status'],\n",
    "                    'lang': result['lang'],\n",
    "                    'entities': result['entities'],\n",
    "                    'urls': result['entities']['urls'],\n",
    "                    'hashtags': result['entities']['hashtags'],\n",
    "                    'user_mentions': result['entities']['user_mentions'],\n",
    "                    'user': result['user']\n",
    "                    }\n",
    "                \n",
    "                try:\n",
    "                    tweets[id_str]['quoted_status_id_str'] = result['quoted_status_id_str']\n",
    "                except KeyError:\n",
    "                    tweets[id_str]['quoted_status_id_str'] = None\n",
    "        \n",
    "        \n",
    "        for t in tweets:\n",
    "    \n",
    "            # Convert hashtags to string\n",
    "            n = []\n",
    "            if tweets[t]['hashtags']:\n",
    "                for i in tweets[t]['hashtags']:\n",
    "                    n.append(i['text'].lower())\n",
    "                tweets[t]['hashtags'] = \" \".join(n)\n",
    "            else:\n",
    "                tweets[t]['hashtags'] = \"\"\n",
    "\n",
    "            # Convert user-mentions to string\n",
    "            n = []\n",
    "            if tweets[t]['user_mentions']:\n",
    "                for i in tweets[t]['user_mentions']:\n",
    "                    n.append(i['screen_name'].lower())\n",
    "                tweets[t]['user_mentions'] = \" \".join(n)\n",
    "            else:\n",
    "                tweets[t]['user_mentions'] = \"\"\n",
    "                \n",
    "            # Get coordinates if they exist\n",
    "            if tweets[t]['coordinates']:\n",
    "                print(tweets[t]['coordinates'])\n",
    "                pass\n",
    "            else:\n",
    "                tweets[t]['coordinates'] = \"\"\n",
    "                \n",
    "            N.add_node(str(tweets[t]['id_str']), label=tweets[t]['user_screen_name'],\n",
    "                      text=tweets[t]['text'], hashtags=tweets[t]['hashtags'],\n",
    "                      date=tweets[t]['date'], coordinates=tweets[t]['coordinates'])\n",
    "\n",
    "                \n",
    "        # Create edge dict\n",
    "        \n",
    "        edge_dict = {}\n",
    "        \n",
    "        for t in tweets:\n",
    "            temp = []\n",
    "            \n",
    "            # Prep replies to tweets\n",
    "            rtt = tweets[t]['reply_to_tweet']\n",
    "                \n",
    "            # Prep quoted_status_id_str\n",
    "            qis = tweets[t]['quoted_status_id_str']\n",
    "                \n",
    "            # Create edge list keys and weight\n",
    "            \n",
    "            if rtt:\n",
    "                temp = rtt\n",
    "            elif qis:\n",
    "                temp = qis\n",
    "\n",
    "            if temp:\n",
    "                try:\n",
    "                    edge_dict[str([tweets[t]['id_str'], str(temp)])]['weight'] += 1\n",
    "\n",
    "                except KeyError:\n",
    "                    edge_dict[str([tweets[t]['id_str'], str(temp)])] = {}\n",
    "                    edge_dict[str([tweets[t]['id_str'], str(temp)])]['node'] = tweets[t]['id_str']\n",
    "                    edge_dict[str([tweets[t]['id_str'], str(temp)])]['target'] = str(temp)\n",
    "                    edge_dict[str([tweets[t]['id_str'], str(temp)])]['weight'] = 1\n",
    "\n",
    "            # Add edges for @mentions and replies to users\n",
    "            for e in edge_dict:\n",
    "                N.add_edge(edge_dict[e]['node'],\n",
    "                           edge_dict[e]['target'],\n",
    "                           weight=edge_dict[e]['weight'])\n",
    "        \n",
    "            \n",
    "        # Insert Data analysis here\n",
    "        print(\"Nodes: {}, Edges: {}\".format(len(N.nodes()), len(N.edges())))\n",
    "        \n",
    "    # Write N graph in gexf for Gephi\n",
    "    file_name = \"{}_tweets_graph_{}.gexf\".format(\n",
    "        convert(\"_\".join(search_queries)),\n",
    "        datetime.datetime.now())\n",
    "    \n",
    "    nx.write_gexf(N, os.path.join(save_dir, file_name))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up Graph\n",
    "N = nx.DiGraph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nodes: 226, Edges: 33\n",
      "Nodes: 265, Edges: 39\n"
     ]
    }
   ],
   "source": [
    "twitter_data_to_graph(search_queries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### That's it!  Now open Gephi and go play with your graph!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}