Облако слов из новостей или диалогов vk.com

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from PIL import Image
from os import path
from wordcloud import WordCloud
import pymorphy2
import time
import re
import numpy as np
import random
import pandas as pd

from nltk.corpus import stopwords
from collections import Counter
import re
import vk


# -31480508 pikabu - You can see it as group id in browser bar
# -20629724 habrahabr
# -34274053 moscow
# -129636704 space (#ВКосмосе)
# -77270571 geektimes
# -46252034 naked science
# -227 MSU

access_token = "you_token_hear"
session = vk.Session(access_token=access_token)
vkapi = vk.API(session, v='5.71')

def log_write(feature, value=None):
    '''
    :param feature: name of logging feature
    :param value: the value of logging feature
    :return: None
    '''
    # Write data to log file
    f_log = open('/home/username/scripts/vk_log.txt', 'a')
    f_log.write(feature + value)
    f_log.close()


def getLikes(user_id, cnt, vkapi):
    '''
    :param user_id: vk user id
    :param cnt: number of post in thousand
    :param vkapi: vk API object
    :return: posts data dictionary
    '''
    subscriptions_list = vkapi.users.getSubscriptions(user_id=user_id,extended=0)['groups']['items']
    # we form the list of id, which needs to be passed in the following method
    groups_list = ['-' + str(x) for x in subscriptions_list]
    # we form newsfeed
    all_newsfeeds = []
    newsfeed = None
    for c in range(cnt):
        if c == 0:
            kwargs = {
                'filters': 'post',
                'source_ids': ', '.join(groups_list),
                'count': 100,
            }
            newsfeed = vkapi.newsfeed.get(**kwargs)
        else:
            next_from = newsfeed['next_from']
            kwargs = {
                'start_from': next_from,
                'filters': 'post',
                'source_ids': ', '.join(groups_list),
                'count': 100,
            }
            newsfeed = vkapi.newsfeed.get(**kwargs)
        all_newsfeeds.append(newsfeed['items'])
        time.sleep(1)
    # Process all news with its keys (json structure)
    all_keys = []
    for portion in all_newsfeeds:  # portion of news (100)
        for post in portion:  # each news in portion
            for key in post:
                if key not in all_keys: all_keys.append(key)
    # set output dictionary structure
    post_data = {}
    for key in all_keys:
        post_data[key] = []
    # Collect data from newsfeeds
    for portion in all_newsfeeds:
        for post in portion:
            try:
                for key in post_data:
                    if key in post:
                        post_data[key] += [post[key]]
                    elif key not in post:
                        post_data[key] += ['None']
            except KeyError as var:
                pass
    return post_data

# Start log file
log_write('', str(time.asctime()) + '\t')

user_id =  # your id (int)
all_data = getLikes(user_id, 2, vkapi)  # Scan 2 * 100 = 200 news posts

log_write('\t message download: ', str(len(all_data['text'])))

# If need human-readable data
# import datetime
# for data in all_data['date']:
#     print datetime.datetime.fromtimestamp(
#         int(data)
#     ).strftime('%Y-%m-%d %H:%M:%S')

# Add data to exist df and save it
len_old = 0
len_new = 0
try:
    final_df_new = pd.DataFrame.from_dict(all_data, orient='columns')
    final_df_old = pd.read_pickle('/home/username/scripts/vk_news_df')
    len_old = len(final_df_old.index)
    final_df_old = final_df_old.append(final_df_new, ignore_index=True)
    final_df_old.drop_duplicates(subset=['post_id'], inplace=True)
    final_df = final_df_old.copy()
    len_new = len(final_df.index)
# write number of new mwssage into log file
    log_write('\t new news message: ', str(len_new - len_old))
except IOError:
    # Create new file
    final_df = pd.DataFrame.from_dict(all_data, orient='columns')
    log_write('\t new news message: ', str(len(final_df.index)))

    # Save df to file
final_df.to_pickle('/home/username/scripts/vk_news_df')

news_to_process = 200
ind_all = final_df.index
all_text = final_df.loc[ind_all[-news_to_process:], ['text']]
tag_re = re.compile(r'(<!--.*?-->|<[^>]*>)')
# Remove well-formed tags, fixing mistakes by legitimate users
# clear each post from tag and whitespaces
all_text = [re.sub(' +',' ',tag_re.sub(' ', x[0])) for x in all_text.values]
list_in = [all_text]
list_out = ['']

for t in range(len(list_in)):
    self_messages = list_in[t]
    str_data = ' '.join(self_messages)
    str_data = str_data.lower()
    def checkGood(symb):
        good1 = 'ёйцукенгшщзхъэждлорпавыфячсмитьбю'.decode('utf-8')
        good2 = u'ёйцукенгшщзхъэждлорпавыфячсмитьбю'
        if symb in good1:
            return True
        elif symb in good2:
            return True
        else:
            return False
    text = ''
    for i in str_data:
        if i == ' ' or i == '\n':
            text += ' '
        else:
            if checkGood(i):
                text += i
            else:
                text += ''
    text = re.sub(' +',' ', text)    
    str_data = text[:]
    # Normalize of word form
    morph = pymorphy2.MorphAnalyzer()
    text = ''
    for i in str_data.split(' '):
        p = morph.parse(i)[0]
        text += p.normal_form + ' '
    str_data = text[:]
    # Stop words check
    stop_words = stopwords.words('russian')
    stop_words.extend([u'что', u'это', u'так',
                       u'вот', u'быть', u'как',
                       u'в', u'—', u'к',
                       u'на', u'ок', u'кстати',
                       u'который', u'мочь', u'весь',
                       u'еще', u'также', u'свой',
                       u'ещё', u'самый', u'ул', u'комментарий',
                       u'английский', u'язык'])
    words = str_data.split(' ')
    w_before = len(words)
    words = [i for i in words if i not in stop_words]
    w_after = len(words)
    log_write('\t raw and tidy words: ', str([w_before, w_after]) + '\n')
    str_data = ' '.join(words)
    list_out[t] = str_data[:]

str_news = list_out[0]
# Create wordcloud
alice_mask = np.array(Image.open("/home/username/scripts/gear2.png"))
wc = WordCloud(background_color="white", mask=alice_mask, collocations=False)
wc.generate(str_news)

def grey_color_func(word, font_size, position, orientation, random_state=None, **kwargs):
    #50 shades of white
    return "hsl(0, 0%%, %d%%)" % random.randint(60, 100)

default_colors = wc.to_array()
wc2 = wc.recolor(color_func=grey_color_func, random_state=3)
# store to file
wc2.to_file("/home/username/conky/scripts/news_raw.png")

Menu — Startup Application — Add:
Name: vk_news_reader
Command: sh -c «sleep 600 && /FULL_PATH/vk_news.py»

import numpy as np
import scipy
import nltk
import pandas as pd
import seaborn
import matplotlib as mpl
import matplotlib.pyplot as plt
from PIL import Image
from os import path
from wordcloud import WordCloud
import pymorphy2
import time
import re
import vk_log
import pickle
from collections import Counter


session = vk.Session(access_token='you_token')
vkapi = vk.API(session)

friends = vkapi('friends.get')  # take list of all friends of user
# friends = [1111111, 2222222, 33333333]  # also, we can manual set list of friends use they ID

def get_dialogs(user_id):
    # Get dialog with user
    dialogs = vkapi('messages.getDialogs', user_id=user_id)
    return dialogs

def get_history(friends, sleep_time=0.4):
    # Get all dialogues history
    all_history = []
    i = 0
    for friend in friends:
        friend_dialog = get_dialogs(friend)
        time.sleep(sleep_time)
        dialog_len = friend_dialog[0]
        friend_history = []
        if dialog_len > 200:
            # vk API condition: len <= 200
            resid = dialog_len
            offset = 0
            while resid > 0:
                friend_history += vkapi('messages.getHistory',
                    user_id=friend,
                    count=200,
                    offset=offset)
                time.sleep(sleep_time)
                resid -= 200
                offset += 200
                if resid > 0:
                    print('--processing ', friend, ': ', resid,
                        ' of ', dialog_len, ' messages left')
            all_history += friend_history
        i +=1
        print('processed ', i, ' friends of ', len(friends))
    return all_history

all_history = get_history(friends)

# Save or load data
pickle.dump(all_history, open("all_vk_history.p", "wb"))
# all_history = pickle.load(open("all_vk_history.p", "rb"))

interesting_id = user_id  # set interesting user id

def get_messages_for_user(data, user_id):
    # Extract all message for set user
    user_messages = []
    my_messages = []
    for dialog in data:
        if type(dialog) == dict:
            if dialog['from_id'] == user_id:
                m_text = re.sub("<br>", " ", dialog['body'])
                user_messages.append(m_text)
            elif dialog['from_id'] == YOUR_ID:
                m_text = re.sub("<br>", " ", dialog['body'])
                my_messages.append(m_text)
    print 'Extracted', len(user_messages), ' user messages in total'
    print 'Extracted', len(my_messages), ' my messages in total'
    return user_messages, my_messages

user_messages, my_messages = get_messages_for_user(all_history, interesting_id)

list_in = [user_messages, my_messages]
list_out = ['','']

for t in range(len(list_in)):
    self_messages = list_in[t]
    str_data = ' '.join(self_messages)
    str_data = str_data.lower()

    def checkGood(symb):
        good1 = 'ёйцукенгшщзхъэждлорпавыфячсмитьбю'.decode('utf-8')
        good2 = u'ёйцукенгшщзхъэждлорпавыфячсмитьбю'
        if symb in good1:
            return True
        elif symb in good2:
            return True
        else:
            return False

    text = ''
    for i in str_data:
        if i == ' ' or i == '\n':
            text += ' '
        else:
            if checkGood(i):
                text += i
            else:
                text += ''
    str_data = text[:]

    morph = pymorphy2.MorphAnalyzer()
    text = ''
    for i in str_data.split(' '):
        p = morph.parse(i)[0]
        text += p.normal_form + ' '
    str_data = text[:]

    # Stop words check
    from nltk.corpus import stopwords
    stop_words = stopwords.words('russian')
    stop_words.extend([u'что', u'это', u'так', u'вот', u'быть', u'как', u'в', u'—', u'к', u'на', u'ок', u'кстати',
                      u'ещё', u'вообще', u'мб', u'чтоть', u'весь'])
    words = str_data.split(' ')
    words = [i for i in words if i not in stop_words]
    str_data = ' '.join(words)

    list_out[t] = str_data
    print len(str_data.split(' '))

str_user = list_out[0]
str_my = list_out[1]

print 'Dict strong user: ', len(set(str_lisa.split(' '))) / float(len(str_user.split(' '))) * 100.0
print 'Dict strong my: ', len(set(str_my.split(' '))) / float(len(str_my.split(' '))) * 100.0

alice_mask = np.array(Image.open("IMAGE.jpg"))
wc = WordCloud(background_color="black", mask=alice_mask)
# generate word cloud
wc.generate(str_lisa)

# store to file
wc.to_file("User.png")

alice_mask = np.array(Image.open("IMAGE.jpg"))
wc = WordCloud(background_color="black", mask=alice_mask)
# generate word cloud
wc.generate(str_my)

# store to file
wc.to_file("Me.png")