【笔记4】用pandas实现条目数据格式的推荐算法 (基于用户的协同)
'''
基于用户的协同推荐
条目数据
'''
import pandas as pd
from io import StringIO
import json
#数据类型一:条目(用户、商品、打分)(避免巨型稀疏矩阵)
csv_txt = '''"Angelica","Blues Traveler",3.5
"Angelica","Broken Bells",2.0
"Angelica","Norah Jones",4.5
"Angelica","Phoenix",5.0
"Angelica","Slightly Stoopid",1.5
"Angelica","The Strokes",2.5
"Angelica","Vampire Weekend",2.0
"Bill","Blues Traveler",2.0
"Bill","Broken Bells",3.5
"Bill","Deadmau5",4.0
"Bill","Phoenix",2.0
"Bill","Slightly Stoopid",3.5
"Bill","Vampire Weekend",3.0
"Chan","Blues Traveler",5.0
"Chan","Broken Bells",1.0
"Chan","Deadmau5",1.0
"Chan","Norah Jones",3.0
"Chan","Phoenix",5,
"Chan","Slightly Stoopid",1.0
"Dan","Blues Traveler",3.0
"Dan","Broken Bells",4.0
"Dan","Deadmau5",4.5
"Dan","Phoenix",3.0
"Dan","Slightly Stoopid",4.5
"Dan","The Strokes",4.0
"Dan","Vampire Weekend",2.0
"Hailey","Broken Bells",4.0
"Hailey","Deadmau5",1.0
"Hailey","Norah Jones",4.0
"Hailey","The Strokes",4.0
"Hailey","Vampire Weekend",1.0
"Jordyn","Broken Bells",4.5
"Jordyn","Deadmau5",4.0
"Jordyn","Norah Jones",5.0
"Jordyn","Phoenix",5.0
"Jordyn","Slightly Stoopid",4.5
"Jordyn","The Strokes",4.0
"Jordyn","Vampire Weekend",4.0
"Sam","Blues Traveler",5.0
"Sam","Broken Bells",2.0
"Sam","Norah Jones",3.0
"Sam","Phoenix",5.0
"Sam","Slightly Stoopid",4.0
"Sam","The Strokes",5.0
"Veronica","Blues Traveler",3.0
"Veronica","Norah Jones",5.0
"Veronica","Phoenix",4.0
"Veronica","Slightly Stoopid",2.5
"Veronica","The Strokes",3.0'''
#数据类型二:json数据(用户、商品、打分)
json_txt = '''{"Angelica": {"Blues Traveler": 3.5, "Broken Bells": 2.0,
"Norah Jones": 4.5, "Phoenix": 5.0,
"Slightly Stoopid": 1.5,
"The Strokes": 2.5, "Vampire Weekend": 2.0},
"Bill":{"Blues Traveler": 2.0, "Broken Bells": 3.5,
"Deadmau5": 4.0, "Phoenix": 2.0,
"Slightly Stoopid": 3.5, "Vampire Weekend": 3.0},
"Chan": {"Blues Traveler": 5.0, "Broken Bells": 1.0,
"Deadmau5": 1.0, "Norah Jones": 3.0, "Phoenix": 5,
"Slightly Stoopid": 1.0},
"Dan": {"Blues Traveler": 3.0, "Broken Bells": 4.0,
"Deadmau5": 4.5, "Phoenix": 3.0,
"Slightly Stoopid": 4.5, "The Strokes": 4.0,
"Vampire Weekend": 2.0},
"Hailey": {"Broken Bells": 4.0, "Deadmau5": 1.0,
"Norah Jones": 4.0, "The Strokes": 4.0,
"Vampire Weekend": 1.0},
"Jordyn": {"Broken Bells": 4.5, "Deadmau5": 4.0,
"Norah Jones": 5.0, "Phoenix": 5.0,
"Slightly Stoopid": 4.5, "The Strokes": 4.0,
"Vampire Weekend": 4.0},
"Sam": {"Blues Traveler": 5.0, "Broken Bells": 2.0,
"Norah Jones": 3.0, "Phoenix": 5.0,
"Slightly Stoopid": 4.0, "The Strokes": 5.0},
"Veronica": {"Blues Traveler": 3.0, "Norah Jones": 5.0,
"Phoenix": 4.0, "Slightly Stoopid": 2.5,
"The Strokes": 3.0}
}'''
df = None
#方式一:加载csv数据
def load_csv_txt():
global df
df = pd.read_csv(StringIO(csv_txt), header=None, names=['user','goods','rate'])
#方式二:加载json数据(把json读成条目)
def load_json_txt():
global df
#由json数据得到字典
users = json.loads(json_txt)
#遍历字典,得到条目
csv_txt_ = ''
for user in users:
for goods in users[user]:
csv_txt_ += '{},{},{}\n'.format(user, goods, users[user][goods])
df = pd.read_csv(StringIO(csv_txt_), header=None, names=['user','goods','rate'])
print('测试:读取数据')
#load_csv_txt()
load_json_txt()
def build_xy(user_name1, user_name2):
df1 = df.ix[df['user'] == user_name1, ['goods','rate']]
df2 = df.ix[df['user'] == user_name2, ['goods','rate']]
df3 = pd.merge(df1, df2, on='goods', how='inner') #只保留两人都有评分的商品的评分
return df3['rate_x'], df3['rate_y'] #merge之后默认的列名:rate_x,rate_y
#曼哈顿距离
def manhattan(user_name1, user_name2):
x, y = build_xy(user_name1, user_name2)
return sum(abs(x - y))
#欧几里德距离
def euclidean(user_name1, user_name2):
x, y = build_xy(user_name1, user_name2)
return sum((x - y)**2)**0.5
#闵可夫斯基距离
def minkowski(user_name1, user_name2, r):
x, y = build_xy(user_name1, user_name2)
return sum(abs(x - y)**r)**(1/r)
#皮尔逊相关系数
def pearson(user_name1, user_name2):
x, y = build_xy(user_name1, user_name2)
mean1, mean2 = x.mean(), y.mean()
#分母
denominator = (sum((x-mean1)**2)*sum((y-mean2)**2))**0.5
return [sum((x-mean1)*(y-mean2))/denominator, 0][denominator == 0]
#余弦相似度(数据的稀疏性问题,在文本挖掘中应用得较多)
def cosine(user_name1, user_name2):
x, y = build_xy(user_name1, user_name2)
#分母
denominator = (sum(x*x)*sum(y*y))**0.5
return [sum(x*y)/denominator, 0][denominator == 0]
metric_funcs = {
'manhattan': manhattan,
'euclidean': euclidean,
'minkowski': minkowski,
'pearson': pearson,
'cosine': cosine
}
print('\n测试:计算Angelica与Bill的曼哈顿距离')
print(manhattan('Angelica','Bill'))
#计算最近的邻居(返回:pd.Series)
def computeNearestNeighbor(user_name, metric='pearson', k=3, r=2):
'''
metric: 度量函数
k: 返回k个邻居
r: 闵可夫斯基距离专用
返回:pd.Series,其中index是邻居名称,values是距离
'''
array = df[df['user'] != user_name]['user'].unique()
if metric in ['manhattan', 'euclidean']:
return pd.Series(array, index=array.tolist()).apply(metric_funcs[metric], args=(user_name,)).nsmallest(k)
elif metric in ['minkowski']:
return pd.Series(array, index=array.tolist()).apply(metric_funcs[metric], args=(user_name, r,)).nsmallest(k)
elif metric in ['pearson', 'cosine']:
return pd.Series(array, index=array.tolist()).apply(metric_funcs[metric], args=(user_name,)).nlargest(k)
print('\n测试:计算Hailey的最近邻居')
print(computeNearestNeighbor('Hailey'))
#向给定用户推荐(返回:pd.DataFrame)
def recommend(user_name):
"""返回推荐结果列表"""
# 找到距离最近的用户名
nearest_username = computeNearestNeighbor(user_name).index[0]
# 找出这位用户评价过、但自己未曾评价的乐队
df1 = df.ix[df['user'] == user_name, ['goods', 'rate']]
df2 = df.ix[df['user'] == nearest_username, ['goods', 'rate']]
df3 = pd.merge(df1, df2, on='goods', how='outer')
return df3.ix[(df3['rate_x'].isnull()) & (df3['rate_y'].notnull()), ['goods', 'rate_y']].sort_values(by='rate_y')
print('\n测试:为Hailey做推荐')
print(recommend('Hailey'))
#向给定用户推荐(返回:pd.Series)
def recommend2(user_name, metric='pearson', k=3, n=5, r=2):
'''
metric: 度量函数
k: 根据k个最近邻居,协同推荐
r: 闵可夫斯基距离专用
n: 推荐的商品数目
返回:pd.Series,其中index是商品名称,values是加权评分
'''
# 找到距离最近的k个邻居
nearest_neighbors = computeNearestNeighbor(user_name, metric='pearson', k=k, r=r)
# 计算权值
if metric in ['manhattan', 'euclidean', 'minkowski']: # 距离越小,越类似
nearest_neighbors = 1 / nearest_neighbors # 所以,取倒数(或者别的减函数,如:y=2**-x)
elif metric in ['pearson', 'cosine']: # 距离越大,越类似
pass
nearest_neighbors = nearest_neighbors / nearest_neighbors.sum() #已经变为权值
# 逐个邻居找出其评价过、但自己未曾评价的乐队(或商品)的评分,并乘以权值
neighbors_rate_with_weight = []
for neighbor_name in nearest_neighbors.index:
# 每个结果:pd.Series,其中index是商品名称,values是评分(已乘权值)
df1 = df.ix[df['user'] == user_name, ['goods', 'rate']]
df2 = df.ix[df['user'] == neighbor_name, ['goods', 'rate']]
df3 = pd.merge(df1, df2, on='goods', how='outer')
df4 = df3.ix[(df3['rate_x'].isnull()) & (df3['rate_y'].notnull()), ['goods', 'rate_y']]
#注意这中间有一个转化为pd.Series的操作!
neighbors_rate_with_weight.append(pd.Series(df4['rate_y'].tolist(), index=df4['goods']) * nearest_neighbors[neighbor_name])
# 把邻居们的加权评分拼接成pd.DataFrame,按列累加,取最大的前n个商品的评分
return pd.concat(neighbors_rate_with_weight, axis=1).sum(axis=1, skipna=True).nlargest(n) # 黑科技!
print('\n测试:为Hailey做推荐')
print(recommend2('Hailey', metric='manhattan', k=3, n=5))
print('\n测试:为Hailey做推荐')
print(recommend2('Hailey', metric='euclidean', k=3, n=5, r=2))
print('\n测试:为Hailey做推荐')
print(recommend2('Hailey', metric='pearson', k=1, n=5))
【笔记4】用pandas实现条目数据格式的推荐算法 (基于用户的协同)的更多相关文章
- 【笔记3】用pandas实现矩阵数据格式的推荐算法 (基于用户的协同)
原书作者使用字典dict实现推荐算法,并且惊叹于18行代码实现了向量的余弦夹角公式. 我用pandas实现相同的公式只要3行. 特别说明:本篇笔记是针对矩阵数据,下篇笔记是针对条目数据. ''' 基于 ...
- 【笔记6】用pandas实现条目数据格式的推荐算法 (基于物品的协同)
''' 基于物品的协同推荐 矩阵数据 说明: 1.修正的余弦相似度是一种基于模型的协同过滤算法.我们前面提过,这种算法的优势之 一是扩展性好,对于大数据量而言,运算速度快.占用内存少. 2.用户的评价 ...
- 【笔记5】用pandas实现矩阵数据格式的推荐算法 (基于物品的协同)
''' 基于物品的协同推荐 矩阵数据 说明: 1.修正的余弦相似度是一种基于模型的协同过滤算法.我们前面提过,这种算法的优势之 一是扩展性好,对于大数据量而言,运算速度快.占用内存少. 2.用户的评价 ...
- R语言实现关联规则与推荐算法(学习笔记)
R语言实现关联规则 笔者前言:以前在网上遇到很多很好的关联规则的案例,最近看到一个更好的,于是便学习一下,写个学习笔记. 1 1 0 0 2 1 1 0 0 3 1 1 0 1 4 0 0 0 0 5 ...
- 学习笔记-menusript控件中条目权限设置使用
在做一个小程序的时候,偶然发现了使用menusript控件做权限设置的方法,仅此标记,以供参考. 首先创建一个实例:testuseright.sln, 在项目文件里创建两个窗体:Form1.cs和us ...
- Python笔记 #12# Dictionary & Pandas: Object Creation
Document of Dictionaries 10 Minutes to pandas tutorialspoint import pandas as pd data = [['Alex',10] ...
- Office365学习笔记—Lookup类型加载条目过多解决方案
1,随着接触的项目越来越多,遇到的各种奇葩的问题也越来越多,不得不说,SharePoint是个好东西,提高了开发效率,简化了很多基础的功能.但是令人头疼的问题是,当你想做个稍微复杂点的功能,就不得不研 ...
- Office365学习笔记—列表查询,删除条目,更新条目。
1,基于Query语句的列表查询. function retrieveListItems(itemId) { var siteUrl=_spPageContextInfo.webServerRelat ...
- 读书笔记一、pandas之series
转自 # 直接传入一组数据 from pandas import Series, DataFrame obj = Series([4, 2, 3]) obj 0 4 1 2 2 3 dtype: in ...
随机推荐
- [翻译]用 Puppet 搭建易管理的服务器基础架构(3)
我通过伯乐在线翻译了一个Puppet简明教程,一共分为四部分,这是第三部分. 本文由 伯乐在线 - Wing 翻译,黄利民 校稿.未经许可,禁止转载!英文出处:Manuel Kiessling.欢迎加 ...
- CSS笔记之伪类与伪元素
伪类分为两种:UI伪类 与 结构化伪类 UI伪类:a:link{} a:hover{} a:active{} a:visited{} input[type='text']:focus{} ...
- jquery attr()方法
在JS中设置节点的属性与属性值用到setAttribute(),获得节点的属性与属性值用到getAttribute(),而在jquery中,用一个attr()就可以全部搞定了,赞一个先 ^^ jque ...
- arcgis api for flex之专题图制作(饼状图,柱状图等)
最近公司给我一个任务,就是利用arcgis api for flex实现在地图上点(业务数据)直接显示饼状图以及柱状图的专题图制作,而不是通过点击点显示气泡窗口的形式来实现,这个公司已经实现了. 经过 ...
- 配置nginx支持ssl服务器—HTTPS
下文摘自: http://docs.bigbluebutton.org/install/install.html Configuring HTTPS on BigBlueButtonAncho ...
- VS2012 Unit Test —— 我对接口进行单元测试使用的技巧
[题外话] 对单元测试不熟悉的童鞋可参照我之前写过的两篇博文: <在Visual Studio 2012使用单元测试>. <VS2012 单元测试之泛型类(Generics Unit ...
- AFNetworking的理解
AFNetworking的理解 使用方法 1. 新建的工程中导入AFNetworking3.0中的(AFNetworking 和UIKit+AFNetworking两个文件夹) 2. 在用到AFNet ...
- Objective-C内存管理之-引用计数
本文会继续深入学习OC内存管理,内容主要参考iOS高级编程,Objective-C基础教程,疯狂iOS讲义,是我学习内存管理的笔记 内存管理 1 内存管理的基本概念 1.1 Objective-C中的 ...
- Android自定义控件(二)
这一篇主要来讲一下自定义控件中的自定义viewgroup,我们以项目中最常用的下拉刷新和加载更多组件为例 简单介绍一下自定义viewgroup时应该怎么做. 分析:下拉刷新和加载更多的原理和步骤 自定 ...
- 学习 AppFuse
1.Appfuse是个什么鬼? AppFuse是一个集成了当前最流行的Web应用框架的一个更高层次的Web开发框架.换句话说,AppFuse就是一个完整的各主流框架的整合版本.AppFuse总是能够紧 ...