diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0a8b03ec --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +codes/news_rec_web/Vue-newsinfo/node_modules diff --git a/codes/news_rec_server/README.md b/codes/news_rec_server/README.md new file mode 100644 index 00000000..33c69f0d --- /dev/null +++ b/codes/news_rec_server/README.md @@ -0,0 +1,25 @@ + +该项目主要是新闻推荐系统的后端及推荐服务相关的内容，这个项目将前端单独拿出来了。 + +代码规范，先用python规范就行 + +# TODO +项目目录结构待完善 + + +## 前端展示逻辑： +1. 开机页（放一张和app大小相同的页面，上面显示几个字），此时只能点击我的进行登录，否则无法看到内部的具体内容 +2. 登录页，用户名，密码，登录，注册等相关界面 + 输入用户名和密码，如果后端返回ok, 就跳转到推荐页，否则提示账号或者密码错误 +3. 推荐和热门面显示的内容不需要变 +4. 点击某一篇文章之后，向后端发送 user_id, news_id, action= { read, likes, collections } + + +## 后端数据相关 +1. log日志 +2. 用户画像数据及更新 +3. 新闻画像数据及更新 + + +运行新闻推荐后端服务 +python server.py \ No newline at end of file diff --git a/codes/news_rec_server/conf/README.md b/codes/news_rec_server/conf/README.md new file mode 100644 index 00000000..86fd83d1 --- /dev/null +++ b/codes/news_rec_server/conf/README.md @@ -0,0 +1,4 @@ + +# 目录介绍 + +该目录主要存储的是新闻推荐系统的所有配置参数及文件 \ No newline at end of file diff --git a/codes/news_rec_server/conf/__pycache__/config.cpython-38.pyc b/codes/news_rec_server/conf/__pycache__/config.cpython-38.pyc new file mode 100644 index 00000000..ca130972 Binary files /dev/null and b/codes/news_rec_server/conf/__pycache__/config.cpython-38.pyc differ diff --git a/codes/news_rec_server/conf/__pycache__/daoConfig.cpython-38.pyc b/codes/news_rec_server/conf/__pycache__/daoConfig.cpython-38.pyc new file mode 100644 index 00000000..ec6b10b5 Binary files /dev/null and b/codes/news_rec_server/conf/__pycache__/daoConfig.cpython-38.pyc differ diff --git a/codes/news_rec_server/conf/__pycache__/dao_config.cpython-38.pyc b/codes/news_rec_server/conf/__pycache__/dao_config.cpython-38.pyc new file mode 100644 index 00000000..6410ae89 Binary files /dev/null and b/codes/news_rec_server/conf/__pycache__/dao_config.cpython-38.pyc differ diff --git a/codes/news_rec_server/conf/__pycache__/db.cpython-38.pyc b/codes/news_rec_server/conf/__pycache__/db.cpython-38.pyc new file mode 100644 index 00000000..a4b918a9 Binary files /dev/null and b/codes/news_rec_server/conf/__pycache__/db.cpython-38.pyc differ diff --git a/codes/news_rec_server/conf/dao_config.py b/codes/news_rec_server/conf/dao_config.py new file mode 100644 index 00000000..2f9d63e0 --- /dev/null +++ b/codes/news_rec_server/conf/dao_config.py @@ -0,0 +1,40 @@ +# 数据库相关的配置文件 +user_info_db_name = "userinfo" # 用户数据相关的数据库 +register_user_table_name = "register_user" # 注册用户数据表 +user_likes_table_name = "user_likes" # 用户喜欢数据表 +user_collections_table_name = "user_collections" # 用户收藏数据表 +user_read_table_name = "user_read" # 用户阅读数据表 +exposure_table_name_prefix = "exposure" # 用户曝光数据表的前缀 + +# log数据，每天都会落一个盘，并由时间信息进行命名 +loginfo_db_name = "loginfo" # log数据库 +loginfo_table_name_prefix = "log" # log数据表的前缀 + +# 默认配置 +mysql_username = "root" +mysql_passwd = "123456" +mysql_hostname = "localhost" +mysql_port = "3306" + +# MongoDB +mongo_hostname = "127.0.0.1" +mongo_port = 27017 +# Sina原始数据 +sina_db_name= "SinaNews" +sina_collection_name_prefix= "news" +# 物料池db name +material_db_name = "NewsRecSys" + +# 特征画像 集合名称 +feature_protrail_collection_name = "FeatureProtrail" +redis_mongo_collection_name = "RedisProtrail" +user_protrail_collection_name = "UserProtrail" + +# Redis +redis_hostname = "127.0.0.1" +redis_port = 6379 + +reclist_redis_db_num = 0 +static_news_info_db_num = 1 +dynamic_news_info_db_num = 2 +user_exposure_db_num = 3 diff --git a/codes/news_rec_server/conf/stop_words.txt b/codes/news_rec_server/conf/stop_words.txt new file mode 100644 index 00000000..f61581c4 --- /dev/null +++ b/codes/news_rec_server/conf/stop_words.txt @@ -0,0 +1,746 @@ +$ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +? +_ +“ +” +、 +。 +《 +》 +一 +一些 +一何 +一切 +一则 +一方面 +一旦 +一来 +一样 +一般 +一转眼 +万一 +上 +上下 +下 +不 +不仅 +不但 +不光 +不单 +不只 +不外乎 +不如 +不妨 +不尽 +不尽然 +不得 +不怕 +不惟 +不成 +不拘 +不料 +不是 +不比 +不然 +不特 +不独 +不管 +不至于 +不若 +不论 +不过 +不问 +与 +与其 +与其说 +与否 +与此同时 +且 +且不说 +且说 +两者 +个 +个别 +临 +为 +为了 +为什么 +为何 +为止 +为此 +为着 +乃 +乃至 +乃至于 +么 +之 +之一 +之所以 +之类 +乌乎 +乎 +乘 +也 +也好 +也罢 +了 +二来 +于 +于是 +于是乎 +云云 +云尔 +些 +亦 +人 +人们 +人家 +什么 +什么样 +今 +介于 +仍 +仍旧 +从 +从此 +从而 +他 +他人 +他们 +以 +以上 +以为 +以便 +以免 +以及 +以故 +以期 +以来 +以至 +以至于 +以致 +们 +任 +任何 +任凭 +似的 +但 +但凡 +但是 +何 +何以 +何况 +何处 +何时 +余外 +作为 +你 +你们 +使 +使得 +例如 +依 +依据 +依照 +便于 +俺 +俺们 +倘 +倘使 +倘或 +倘然 +倘若 +借 +假使 +假如 +假若 +傥然 +像 +儿 +先不先 +光是 +全体 +全部 +兮 +关于 +其 +其一 +其中 +其二 +其他 +其余 +其它 +其次 +具体地说 +具体说来 +兼之 +内 +再 +再其次 +再则 +再有 +再者 +再者说 +再说 +冒 +冲 +况且 +几 +几时 +凡 +凡是 +凭 +凭借 +出于 +出来 +分别 +则 +则甚 +别 +别人 +别处 +别是 +别的 +别管 +别说 +到 +前后 +前此 +前者 +加之 +加以 +即 +即令 +即使 +即便 +即如 +即或 +即若 +却 +去 +又 +又及 +及 +及其 +及至 +反之 +反而 +反过来 +反过来说 +受到 +另 +另一方面 +另外 +另悉 +只 +只当 +只怕 +只是 +只有 +只消 +只要 +只限 +叫 +叮咚 +可 +可以 +可是 +可见 +各 +各个 +各位 +各种 +各自 +同 +同时 +后 +后者 +向 +向使 +向着 +吓 +吗 +否则 +吧 +吧哒 +吱 +呀 +呃 +呕 +呗 +呜 +呜呼 +呢 +呵 +呵呵 +呸 +呼哧 +咋 +和 +咚 +咦 +咧 +咱 +咱们 +咳 +哇 +哈 +哈哈 +哉 +哎 +哎呀 +哎哟 +哗 +哟 +哦 +哩 +哪 +哪个 +哪些 +哪儿 +哪天 +哪年 +哪怕 +哪样 +哪边 +哪里 +哼 +哼唷 +唉 +唯有 +啊 +啐 +啥 +啦 +啪达 +啷当 +喂 +喏 +喔唷 +喽 +嗡 +嗡嗡 +嗬 +嗯 +嗳 +嘎 +嘎登 +嘘 +嘛 +嘻 +嘿 +嘿嘿 +因 +因为 +因了 +因此 +因着 +因而 +固然 +在 +在下 +在于 +地 +基于 +处在 +多 +多么 +多少 +大 +大家 +她 +她们 +好 +如 +如上 +如上所述 +如下 +如何 +如其 +如同 +如是 +如果 +如此 +如若 +始而 +孰料 +孰知 +宁 +宁可 +宁愿 +宁肯 +它 +它们 +对 +对于 +对待 +对方 +对比 +将 +小 +尔 +尔后 +尔尔 +尚且 +就 +就是 +就是了 +就是说 +就算 +就要 +尽 +尽管 +尽管如此 +岂但 +己 +已 +已矣 +巴 +巴巴 +并 +并且 +并非 +庶乎 +庶几 +开外 +开始 +归 +归齐 +当 +当地 +当然 +当着 +彼 +彼时 +彼此 +往 +待 +很 +得 +得了 +怎 +怎么 +怎么办 +怎么样 +怎奈 +怎样 +总之 +总的来看 +总的来说 +总的说来 +总而言之 +恰恰相反 +您 +惟其 +慢说 +我 +我们 +或 +或则 +或是 +或曰 +或者 +截至 +所 +所以 +所在 +所幸 +所有 +才 +才能 +打 +打从 +把 +抑或 +拿 +按 +按照 +换句话说 +换言之 +据 +据此 +接着 +故 +故此 +故而 +旁人 +无 +无宁 +无论 +既 +既往 +既是 +既然 +时候 +是 +是以 +是的 +曾 +替 +替代 +最 +有 +有些 +有关 +有及 +有时 +有的 +望 +朝 +朝着 +本 +本人 +本地 +本着 +本身 +来 +来着 +来自 +来说 +极了 +果然 +果真 +某 +某个 +某些 +某某 +根据 +欤 +正值 +正如 +正巧 +正是 +此 +此地 +此处 +此外 +此时 +此次 +此间 +毋宁 +每 +每当 +比 +比及 +比如 +比方 +没奈何 +沿 +沿着 +漫说 +焉 +然则 +然后 +然而 +照 +照着 +犹且 +犹自 +甚且 +甚么 +甚或 +甚而 +甚至 +甚至于 +用 +用来 +由 +由于 +由是 +由此 +由此可见 +的 +的确 +的话 +直到 +相对而言 +省得 +看 +眨眼 +着 +着呢 +矣 +矣乎 +矣哉 +离 +竟而 +第 +等 +等到 +等等 +简言之 +管 +类如 +紧接着 +纵 +纵令 +纵使 +纵然 +经 +经过 +结果 +给 +继之 +继后 +继而 +综上所述 +罢了 +者 +而 +而且 +而况 +而后 +而外 +而已 +而是 +而言 +能 +能否 +腾 +自 +自个儿 +自从 +自各儿 +自后 +自家 +自己 +自打 +自身 +至 +至于 +至今 +至若 +致 +般的 +若 +若夫 +若是 +若果 +若非 +莫不然 +莫如 +莫若 +虽 +虽则 +虽然 +虽说 +被 +要 +要不 +要不是 +要不然 +要么 +要是 +譬喻 +譬如 +让 +许多 +论 +设使 +设或 +设若 +诚如 +诚然 +该 +说来 +诸 +诸位 +诸如 +谁 +谁人 +谁料 +谁知 +贼死 +赖以 +赶 +起 +起见 +趁 +趁着 +越是 +距 +跟 +较 +较之 +边 +过 +还 +还是 +还有 +还要 +这 +这一来 +这个 +这么 +这么些 +这么样 +这么点儿 +这些 +这会儿 +这儿 +这就是说 +这时 +这样 +这次 +这般 +这边 +这里 +进而 +连 +连同 +逐步 +通过 +遵循 +遵照 +那 +那个 +那么 +那么些 +那么样 +那些 +那会儿 +那儿 +那时 +那样 +那般 +那边 +那里 +都 +鄙人 +鉴于 +针对 +阿 +除 +除了 +除外 +除开 +除此之外 +除非 +随 +随后 +随时 +随着 +难道说 +非但 +非徒 +非特 +非独 +靠 +顺 +顺着 +首先 +！ +， +： +； +？ \ No newline at end of file diff --git a/codes/news_rec_server/controller/__pycache__/log_controller.cpython-38.pyc b/codes/news_rec_server/controller/__pycache__/log_controller.cpython-38.pyc new file mode 100644 index 00000000..6025bfcc Binary files /dev/null and b/codes/news_rec_server/controller/__pycache__/log_controller.cpython-38.pyc differ diff --git a/codes/news_rec_server/controller/__pycache__/user_action_controller.cpython-38.pyc b/codes/news_rec_server/controller/__pycache__/user_action_controller.cpython-38.pyc new file mode 100644 index 00000000..31457009 Binary files /dev/null and b/codes/news_rec_server/controller/__pycache__/user_action_controller.cpython-38.pyc differ diff --git a/codes/news_rec_server/controller/log_controller.py b/codes/news_rec_server/controller/log_controller.py new file mode 100644 index 00000000..9bb4d223 --- /dev/null +++ b/codes/news_rec_server/controller/log_controller.py @@ -0,0 +1,19 @@ +from dao.mysql_server import MysqlServer +from dao.entity.logitem import LogItem + +import time + + + +class LogController(): + def __init__(self) -> None: + self.log_info_sql_session = MysqlServer().get_loginfo_session() + + def save_one_log(self,log): + try: + self.log_info_sql_session.add(log) + self.log_info_sql_session.commit() + except Exception as e: + print(str(e)) + return False + return True \ No newline at end of file diff --git a/codes/news_rec_server/controller/user_action_controller.py b/codes/news_rec_server/controller/user_action_controller.py new file mode 100644 index 00000000..45273dca --- /dev/null +++ b/codes/news_rec_server/controller/user_action_controller.py @@ -0,0 +1,116 @@ + +from dao.mysql_server import MysqlServer +from dao.entity.user_exposure import UserExposure +from dao.entity.register_user import RegisterUser +from dao.entity.user_likes import UserLikes +from dao.entity.user_read import UserRead +from dao.entity.user_collections import UserCollections + +# 初始化数据表 +user = UserLikes() +user = UserCollections() +user = UserExposure() +user = UserRead() + + +class UserAction(): + def __init__(self) -> None: + self.user_exposure_sql_session = MysqlServer().get_user_exposure_session() + self.register_user_sql_session = MysqlServer().get_register_user_session() + self.user_like_sql_session = MysqlServer().get_user_like_session() + self.user_collection_sql_session = MysqlServer().get_user_collection_session() + self.user_read_sql_session = MysqlServer().get_user_read_session() + + def user_is_exist(self, user, user_type): + """ + 1 表示正确；2 表示密码错误；0 表示用户不存在 + """ + if user_type == "login": + if self.register_user_sql_session.query(RegisterUser).filter(RegisterUser.username == user.username, \ + RegisterUser.passwd == user.passwd).count() > 0: + return 1 + elif self.register_user_sql_session.query(RegisterUser).filter(RegisterUser.username == user.username).count() > 0: + return 2 + else: + return 0 + else: + if self.register_user_sql_session.query(RegisterUser).filter(RegisterUser.username == user.username).count() > 0: + return 1 + else: + return 0 + + def save_user(self,user): + try: + self.register_user_sql_session.add(user) + self.register_user_sql_session.commit() + # self.register_user_sql_session.close() + except Exception as e: + print(str(e)) + return False + return True + + def get_user_id_by_name(self,username): + + try: + userid = self.register_user_sql_session.query(RegisterUser.userid).filter(RegisterUser.username == username).one()[0] + except Exception as e: + print(str(e)) + return None + return userid + + def get_likes_counts_by_user(self,user_id,news_id): + return self.user_like_sql_session.query(UserLikes).filter(UserLikes.userid == user_id, UserLikes.newid == news_id).count() + + def get_coll_counts_by_user(self,user_id,news_id): + return self.user_collection_sql_session.query(UserCollections).filter(UserCollections.userid == user_id, + UserCollections.newid == news_id).count() + + def del_likes_by_user(self,user_id,news_id): + try: + print(user_id,news_id) + delItems = self.user_like_sql_session.query(UserLikes).filter(UserLikes.userid == user_id, UserLikes.newid == news_id) + # print(delItems.count()) + if delItems.count() > 0: + self.user_like_sql_session.query(UserLikes).filter(UserLikes.userid == user_id, UserLikes.newid == news_id).delete() + self.user_like_sql_session.commit() + except Exception as e: + print(str(e)) + return False + return True + + def del_coll_by_user(self,user_id,news_id): + try: + delItems = self.user_collection_sql_session.query(UserCollections).filter(UserCollections.userid == user_id, + UserCollections.newid == news_id) + if delItems.count() > 0: + self.user_collection_sql_session.query(UserCollections).filter(UserCollections.userid == user_id, + UserCollections.newid == news_id).delete() + self.user_collection_sql_session.commit() + except Exception as e: + print(str(e)) + return False + return True + + def save_one_action(self,action): + if isinstance(action, UserLikes): + try: + self.user_like_sql_session.add(action) + self.user_like_sql_session.commit() + except Exception as e: + print(str(e)) + return False + elif isinstance(action, UserCollections): + try: + self.user_collection_sql_session.add(action) + self.user_collection_sql_session.commit() + except Exception as e: + print(str(e)) + return False + elif isinstance(action, UserRead): + try: + self.user_read_sql_session.add(action) + self.user_read_sql_session.commit() + except Exception as e: + print(str(e)) + return False + return True \ No newline at end of file diff --git a/codes/news_rec_server/dao/README.md b/codes/news_rec_server/dao/README.md new file mode 100644 index 00000000..56e28290 --- /dev/null +++ b/codes/news_rec_server/dao/README.md @@ -0,0 +1,16 @@ +DAO层主要是做数据持久层的工作，主要与数据库进行交互。DAO层首先会创建DAO接口，然后会在配置文件中定义该接口的实现类， +接着就可以在模块中就可以调用DAO 的接口进行数据业务的而处理，并且不用关注此接口的具体实现类是哪一个类。DAO 层的数据源和数据库连接的参数数都是在配置文件中进行配置的。 + + +TODO: 设置开机自启动 +启动mongodb（服务器断电之后就会断开链接）： +sudo ./mongod --dbpath=/usr/local/mongodb/data/ --fork --logpath=/usr/local/mongodb/log + +TODO: 设置开机自启动 +启动redis +redis-server --daemonize yes --port 6378 --requirepass 123456 +redis-cli --raw + + +# TODO +MySQL 使用SQLAlchemy 插入中文会报错 diff --git a/codes/news_rec_server/dao/__pycache__/Mongo.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/Mongo.cpython-38.pyc new file mode 100644 index 00000000..55f6fc42 Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/Mongo.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/Mysql.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/Mysql.cpython-38.pyc new file mode 100644 index 00000000..5d9cc18d Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/Mysql.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/Redis.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/Redis.cpython-38.pyc new file mode 100644 index 00000000..db3b03be Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/Redis.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-38.pyc new file mode 100644 index 00000000..aa93fb9a Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-39.pyc b/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-39.pyc new file mode 100644 index 00000000..ce3cae93 Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/mongo_server.cpython-39.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/mysql_server.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/mysql_server.cpython-38.pyc new file mode 100644 index 00000000..4b38f175 Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/mysql_server.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/__pycache__/redis_server.cpython-38.pyc b/codes/news_rec_server/dao/__pycache__/redis_server.cpython-38.pyc new file mode 100644 index 00000000..a9254067 Binary files /dev/null and b/codes/news_rec_server/dao/__pycache__/redis_server.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/logitem.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/logitem.cpython-38.pyc new file mode 100644 index 00000000..e7c42f99 Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/logitem.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/register_user.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/register_user.cpython-38.pyc new file mode 100644 index 00000000..1c51d454 Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/register_user.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/user_collections.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/user_collections.cpython-38.pyc new file mode 100644 index 00000000..9a5e91c9 Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/user_collections.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/user_exposure.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/user_exposure.cpython-38.pyc new file mode 100644 index 00000000..437b2187 Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/user_exposure.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/user_likes.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/user_likes.cpython-38.pyc new file mode 100644 index 00000000..3caaac25 Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/user_likes.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/__pycache__/user_read.cpython-38.pyc b/codes/news_rec_server/dao/entity/__pycache__/user_read.cpython-38.pyc new file mode 100644 index 00000000..ecfc921b Binary files /dev/null and b/codes/news_rec_server/dao/entity/__pycache__/user_read.cpython-38.pyc differ diff --git a/codes/news_rec_server/dao/entity/logitem.py b/codes/news_rec_server/dao/entity/logitem.py new file mode 100644 index 00000000..fa56afbf --- /dev/null +++ b/codes/news_rec_server/dao/entity/logitem.py @@ -0,0 +1,38 @@ +import sys +sys.path.append("../../") +import time +from sqlalchemy import Column, String, Integer,DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql import func + +from dao.mysql_server import MysqlServer +from conf.dao_config import loginfo_db_name, loginfo_table_name_prefix + +# 定义基类 +Base = declarative_base() + +# 定义映射关系 +class LogItem(Base): + """log日志数据 + """ + postfix = time.strftime("%Y_%m_%d", time.localtime()) + + # 每天都会创建一个新的表，带有时间信息 + __tablename__ = '{}_{}'.format(loginfo_table_name_prefix, postfix) + index = Column(Integer(), primary_key=True) + userid = Column(String(30)) + newsid = Column(String(100)) + + # 阅读、点赞、收藏 + actiontype = Column(String(20)) + actiontime = Column(DateTime(timezone=True), server_default=func.now()) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_loginfo_engine() + Base.metadata.create_all(engine) + + def new(self,userid,newsid,actiontype): + self.userid = userid + self.newsid = newsid + self.actiontype = actiontype diff --git a/codes/news_rec_server/dao/entity/register_user.py b/codes/news_rec_server/dao/entity/register_user.py new file mode 100644 index 00000000..430b1641 --- /dev/null +++ b/codes/news_rec_server/dao/entity/register_user.py @@ -0,0 +1,25 @@ +from sqlalchemy import Column, String, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql.sqltypes import BigInteger + +from conf.dao_config import register_user_table_name +from dao.mysql_server import MysqlServer + +Base = declarative_base() + +class RegisterUser(Base): + """用户注册数据 + """ + __tablename__ = register_user_table_name + index = Column(Integer(), primary_key=True) + userid = Column(BigInteger()) + username = Column(String(30)) + passwd = Column(String(500)) + gender = Column(String(30)) + age = Column(String(5)) + city = Column(String(10)) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_register_user_engine() + Base.metadata.create_all(engine) \ No newline at end of file diff --git a/codes/news_rec_server/dao/entity/user_collections.py b/codes/news_rec_server/dao/entity/user_collections.py new file mode 100644 index 00000000..8fcce2eb --- /dev/null +++ b/codes/news_rec_server/dao/entity/user_collections.py @@ -0,0 +1,30 @@ +from sqlalchemy import Column, String, Integer, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql.sqltypes import BigInteger, DateTime + +from conf.dao_config import user_collections_table_name +from dao.mysql_server import MysqlServer +from sqlalchemy.sql import func + +Base = declarative_base() + +class UserCollections(Base): + """用户收藏新闻数据 + """ + __tablename__ = user_collections_table_name + index = Column(Integer(), primary_key=True,autoincrement=True) + userid = Column(BigInteger()) + username = Column(String(30)) + newid = Column(String(100)) + curtime = Column(DateTime(timezone=True), server_default=func.now()) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_user_collection_engine() + Base.metadata.create_all(engine) + + def new(self,userid,username,newid): + self.userid = userid + self.username = username + self.newid = newid + # self.curtime = curtime \ No newline at end of file diff --git a/codes/news_rec_server/dao/entity/user_exposure.py b/codes/news_rec_server/dao/entity/user_exposure.py new file mode 100644 index 00000000..e8d7167e --- /dev/null +++ b/codes/news_rec_server/dao/entity/user_exposure.py @@ -0,0 +1,35 @@ +from sqlalchemy import Column, String, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql.sqltypes import BigInteger,DateTime + +from conf.dao_config import exposure_table_name_prefix +from dao.mysql_server import MysqlServer +from sqlalchemy.sql import func + +import time + +Base = declarative_base() + +class UserExposure(Base): + """用户曝光数据 + """ + postfix = time.strftime("%Y_%m_%d", time.localtime()) + + # 每天都会创建一个新的表，带有时间信息 + __tablename__ = '{}_{}'.format(exposure_table_name_prefix, postfix) + + index = Column(Integer(), primary_key=True,autoincrement=True) + userid = Column(BigInteger()) + newid = Column(String(600)) + curtime = Column(String(50)) + # curtime = Column(DateTime(timezone=True), server_default=func.now()) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_user_exposure_engine() + Base.metadata.create_all(engine) + + def new(self,userid,newid,curtime): + self.userid = userid + self.newid = newid + self.curtime = curtime \ No newline at end of file diff --git a/codes/news_rec_server/dao/entity/user_likes.py b/codes/news_rec_server/dao/entity/user_likes.py new file mode 100644 index 00000000..3ce797ac --- /dev/null +++ b/codes/news_rec_server/dao/entity/user_likes.py @@ -0,0 +1,30 @@ +from sqlalchemy import Column, String, Integer, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql.sqltypes import BigInteger + +from conf.dao_config import user_likes_table_name +from dao.mysql_server import MysqlServer +from sqlalchemy.sql import func + +Base = declarative_base() + +class UserLikes(Base): + """用户喜欢新闻数据 + """ + __tablename__ = user_likes_table_name + index = Column(Integer(), primary_key=True,autoincrement=True) + userid = Column(BigInteger()) + username = Column(String(30)) + newid = Column(String(100)) + curtime = Column(DateTime(timezone=True), server_default=func.now()) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_user_like_engine() + Base.metadata.create_all(engine) + + def new(self,userid,username,newid): + self.userid = userid + self.username = username + self.newid = newid + # self.curtime = curtime \ No newline at end of file diff --git a/codes/news_rec_server/dao/entity/user_read.py b/codes/news_rec_server/dao/entity/user_read.py new file mode 100644 index 00000000..cff2515b --- /dev/null +++ b/codes/news_rec_server/dao/entity/user_read.py @@ -0,0 +1,28 @@ +from sqlalchemy import Column, String, Integer, DateTime +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql.sqltypes import BigInteger, DateTime + +from conf.dao_config import user_read_table_name +from dao.mysql_server import MysqlServer +from sqlalchemy.sql import func + +Base = declarative_base() + +class UserRead(Base): + """用户阅读新闻数据 + """ + __tablename__ = user_read_table_name + index = Column(Integer(), primary_key=True, autoincrement=True) + userid = Column(BigInteger()) + newid = Column(String(100)) + curtime = Column(DateTime(timezone=True), server_default=func.now()) + + def __init__(self): + # 与数据库绑定映射关系 + engine = MysqlServer().get_user_read_engine() + Base.metadata.create_all(engine) + + def new(self, userid, newid, actiontime): + self.userid = userid + self.newid = newid + self.curtime = str(actiontime) \ No newline at end of file diff --git a/codes/news_rec_server/dao/mongo_server.py b/codes/news_rec_server/dao/mongo_server.py new file mode 100644 index 00000000..47105e7b --- /dev/null +++ b/codes/news_rec_server/dao/mongo_server.py @@ -0,0 +1,54 @@ +import sys +import datetime +sys.path.append("../") +import pymongo +from conf.dao_config import mongo_hostname, mongo_port +from conf.dao_config import sina_db_name, sina_collection_name_prefix +from conf.dao_config import material_db_name, feature_protrail_collection_name +from conf.dao_config import redis_mongo_collection_name +from conf.dao_config import user_protrail_collection_name + +class MongoServer(object): + def __init__(self, _mongo_hostname=mongo_hostname, _mongo_port=mongo_port, _sina_db_name=sina_db_name, + _sina_collection_name_prefix=sina_collection_name_prefix, _material_db_name=material_db_name, + _feature_protrail_collection_name=feature_protrail_collection_name, + _redis_mongo_collection_name=redis_mongo_collection_name, + _user_protrail_collection_name=user_protrail_collection_name): + self._hostname = _mongo_hostname + self._port = _mongo_port + self._sina_db_name = _sina_db_name + self._sina_collection_name_prefix = _sina_collection_name_prefix + self._material_db_name = _material_db_name + self._feature_protrail_collection_name = _feature_protrail_collection_name + self._redis_mongo_collection_name = _redis_mongo_collection_name + self._user_protrail_collection_name = user_protrail_collection_name + + self._mongo_client = self._mongodb() + + def _mongodb(self): + """连接mongo数据库，并返回数据库 + """ + client = pymongo.MongoClient(self._hostname, self._port) + return client + + def get_feature_protrail_collection(self): + """特征画像集合 + """ + return self._mongo_client[self._material_db_name][self._feature_protrail_collection_name] + + def get_sina_news_collection(self): + """原始新闻画像集合, 新闻爬取数据collection会以当天的时间命名 + """ + sina_collection_name = self._sina_collection_name_prefix + "_" + \ + "".join(str(datetime.date.today()).split('-')) + return self._mongo_client[self._sina_db_name][sina_collection_name] + + def get_redis_mongo_collection(self): + """redis中的mongo备份数据集合 + """ + return self._mongo_client[self._material_db_name][self._redis_mongo_collection_name] + + def get_user_protrail_collection(self): + """用户画像的数据集合 + """ + return self._mongo_client[self._material_db_name][self._user_protrail_collection_name] diff --git a/codes/news_rec_server/dao/mysql_server.py b/codes/news_rec_server/dao/mysql_server.py new file mode 100644 index 00000000..4be7c53c --- /dev/null +++ b/codes/news_rec_server/dao/mysql_server.py @@ -0,0 +1,102 @@ +import sys +sys.path.append("../") +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from conf.dao_config import loginfo_db_name, user_info_db_name + + +class MysqlServer(object): + def __init__(self, username="root", passwd="123456", hostname="localhost", port="3306", + user_info_db_name=user_info_db_name, loginfo_db_name=loginfo_db_name): + + self.username = username + self.passwd = passwd + self.hostname = hostname + self.port = port + self.user_info_db_name = user_info_db_name + self.loginfo_db_name = loginfo_db_name + + def session(self, db_name): + """链接数据库，绑定映射关系 + """ + # 创建引擎 + engine = create_engine("mysql+pymysql://{}:{}@{}:{}/{}".format( + self.username, self.passwd, self.hostname, self.port, db_name + ), encoding="utf-8", echo=False) + # 创建会话 + session = sessionmaker(bind=engine) + # 返回engine 和 session, 前者用来绑定本地数据，后者用来本地操作数据库 + return engine, session() + + def get_register_user_session(self): + """获取注册用户session + """ + _, sess = self.session(self.user_info_db_name) + return sess + + def get_loginfo_session(self): + """获取log日志的session + """ + _, sess = self.session(self.loginfo_db_name) + return sess + + def get_user_like_session(self): + """获取用户喜欢新闻的session + """ + _, sess = self.session(self.user_info_db_name) + return sess + + def get_user_collection_session(self): + """获取用户收藏新闻的session + """ + _, sess = self.session(self.user_info_db_name) + return sess + + def get_user_exposure_session(self): + """获取用户曝光的session + """ + _, sess = self.session(self.user_info_db_name) + return sess + + def get_user_read_session(self): + """获取用户阅读的session + """ + _, sess = self.session(self.user_info_db_name) + return sess + + def get_register_user_engine(self): + """ + """ + engine, _ = self.session(self.user_info_db_name) + return engine + + def get_loginfo_engine(self): + """ + """ + engine, _ = self.session(self.loginfo_db_name) + return engine + + def get_user_like_engine(self): + """获取用户喜欢新闻的engine + """ + engine, _ = self.session(self.user_info_db_name) + return engine + + def get_user_collection_engine(self): + """获取用户收藏新闻的engine + """ + engine, _ = self.session(self.user_info_db_name) + return engine + + def get_user_read_engine(self): + """获取用户阅读新闻的engine + """ + engine, _ = self.session(self.user_info_db_name) + return engine + + def get_user_exposure_engine(self): + """获取用户曝光的engine + """ + engine, _ = self.session(self.user_info_db_name) + return engine + diff --git a/codes/news_rec_server/dao/redis_server.py b/codes/news_rec_server/dao/redis_server.py new file mode 100644 index 00000000..8f4489fd --- /dev/null +++ b/codes/news_rec_server/dao/redis_server.py @@ -0,0 +1,42 @@ +import sys +sys.path.append("../") +import redis +from conf.dao_config import redis_hostname, redis_port, static_news_info_db_num, dynamic_news_info_db_num, reclist_redis_db_num +from conf.dao_config import user_exposure_db_num + + +class RedisServer(object): + def __init__(self, _redis_hostname=redis_hostname, _port=redis_port, _static_news_info_db_num=static_news_info_db_num, + _dynamic_news_info_db_num = dynamic_news_info_db_num, _reclist_redis_db_num=reclist_redis_db_num, + _user_exposure_db_num=user_exposure_db_num): + self.hostname = _redis_hostname + self.port = _port + self.static_news_info_db_num = _static_news_info_db_num + self.dynamic_news_info_db_num = _dynamic_news_info_db_num + self.reclist_redis_db_num = _reclist_redis_db_num + self.user_exposure_db_num = _user_exposure_db_num + + def _redis_db(self, db_num=0): + res_db = redis.StrictRedis(host=self.hostname, port=self.port, db=db_num, decode_responses=True) + return res_db + + def get_static_news_info_redis(self): + """获取静态新闻信息数据库 + """ + return self._redis_db(self.static_news_info_db_num) + + def get_dynamic_news_info_redis(self): + """获取动态新闻信息数据库 + """ + return self._redis_db(self.dynamic_news_info_db_num) + + def get_reclist_redis(self): + """用户推荐列表redis数据库 + """ + return self._redis_db(self.reclist_redis_db_num) + + def get_exposure_redis(self): + """用户曝光列表redis数据库 + """ + return self._redis_db(self.user_exposure_db_num) + diff --git a/codes/news_rec_server/logs/material_and_user_process.log b/codes/news_rec_server/logs/material_and_user_process.log new file mode 100644 index 00000000..6235512c --- /dev/null +++ b/codes/news_rec_server/logs/material_and_user_process.log @@ -0,0 +1,10 @@ +2021-12-04-00-10-21 +run update_new_items success. +update_dynamic_feature_protrail success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +process_material success. +process_user.py success. +news detail info are saved in redis db. +update_redis success. + diff --git a/codes/news_rec_server/logs/news_bad_cases.log b/codes/news_rec_server/logs/news_bad_cases.log new file mode 100644 index 00000000..8580f40f --- /dev/null +++ b/codes/news_rec_server/logs/news_bad_cases.log @@ -0,0 +1,151 @@ +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +c1ea3624-7e16-41e3-91ca-5f2237c90016 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +c1ea3624-7e16-41e3-91ca-5f2237c90016 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +c1ea3624-7e16-41e3-91ca-5f2237c90016 +d04ec960-fb54-44c4-93d8-27aee82a14a5 +9eb67338-c4dd-4fab-b9f6-dd3d1f635078 +8e744b2e-283e-4880-a35d-010e22f9b6d1 +64a9131f-7bef-4026-af19-a437258b698b +5735d3ba-2ae7-44b0-87b1-a4212042dfd5 +2ec76526-1734-4631-85d5-38b53c289724 +2631c157-4bd1-469e-a69a-5cc40d56087e +0ed4e74d-5133-42fe-b9e9-c2f4a217aae6 +e6feadd0-b0ca-4dad-9cf0-e51d59208741 +de8f34ed-894f-454a-8af5-498cb5bfa416 +d6bfbcb5-e2aa-43af-85c1-a53776ee55da +c83f6e63-3614-46d4-9c3b-56acad2c6053 +ae39b902-7e4c-4392-972d-97d876e09802 +a2b457b0-232f-4175-a618-08bf257bff13 +8ce201a5-a59a-45e2-9c80-d1530213dd76 +5f52e821-b2f1-4328-85f0-62bc8e0b36e7 +328a2cc0-89bf-4626-b9ee-f3ee4c6873f8 +30fb8ac3-7cc8-4666-9aac-c66cd4cd8e20 +148e8b52-5407-4545-9c5a-1745236f8139 +06ab8ab1-0170-4fca-8f7a-900a82872378 +f9f4c879-005a-4d7a-827e-099666396bd4 +c1ea3624-7e16-41e3-91ca-5f2237c90016 diff --git a/codes/news_rec_server/logs/offline_material_process.log b/codes/news_rec_server/logs/offline_material_process.log new file mode 100644 index 00000000..0cde9292 --- /dev/null +++ b/codes/news_rec_server/logs/offline_material_process.log @@ -0,0 +1,74 @@ +2021-11-30-19-03-01 +scrapy crawl sina_spider --pages success. +run python monitor_news.py success. +run update_new_items success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +news detail info are saved in redis db. +material to mongo and redis success. + +2021-11-30-19-08-01 +scrapy crawl sina_spider --pages success. +run python monitor_news.py success. +run update_new_items success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +news detail info are saved in redis db. +material to mongo and redis success. + +material to mongo and redis fail. + +material to mongo and redis fail. + +2021-12-02-09-13-04 +scrapy crawl sina_spider --pages success. +the news nums of news_20211202 collection is 251 and less then 1000. +run python monitor_news.py success. +material to mongo and redis fail. + +material to mongo and redis fail. + +update_dynamic_feature_protrail success. +material to mongo and redis fail. + +update_dynamic_feature_protrail success. +run update_new_items success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +news detail info are saved in redis db. +material to mongo and redis success. + +2021-12-02-23-00-01 +scrapy crawl sina_spider --pages success. +the news nums of news_20211202 collection is 644 and less then 1000. +run python monitor_news.py success. +material to mongo and redis fail. + +material to mongo and redis fail. + +material to mongo and redis fail. + +update_dynamic_feature_protrail success. +run update_new_items success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +news detail info are saved in redis db. +material to mongo and redis success. + +2021-12-03-09-38-39 +scrapy crawl sina_spider --pages success. +the news nums of news_20211203 collection is 659 and less then 1000. +run python monitor_news.py success. +2021-12-03-09-50-04 +scrapy crawl sina_spider --pages success. +run python monitor_news.py success. +update_dynamic_feature_protrail success. +run update_new_items success. +delete RedisProtrail ... +run update_redis_mongo_protrail_data success. +news detail info are saved in redis db. +material to mongo and redis success. + +2021-12-04-00-00-01 +scrapy crawl sina_spider --pages success. +run python monitor_news.py success. diff --git a/codes/news_rec_server/logs/offline_rec_list_to_redis.log b/codes/news_rec_server/logs/offline_rec_list_to_redis.log new file mode 100644 index 00000000..8fc2eccf --- /dev/null +++ b/codes/news_rec_server/logs/offline_rec_list_to_redis.log @@ -0,0 +1,93 @@ +2021-11-30-19-03-01 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-11-30-19-08-19 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-01-01-00-01 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-02-01-00-01 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-02-09-13-30 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-02-09-18-07 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-02-09-23-18 +a sorted news_ids are saved into redis. +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-01-00-02 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-09-32-44 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-09-33-10 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-09-33-54 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-05-18 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-13-03 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-14-12 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-18-59 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-22-57 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-27-21 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-28-49 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-03-10-45-22 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + +2021-12-04-00-11-59 +a sorted news_ids are saved into redis. +a hot rec list are saved into redis..... +run /home/recsys/miniconda3/envs/news_rec_py3/bin/python /home/recsys/news_rec_server/recprocess/offline.py success. + diff --git a/codes/news_rec_server/materials/README.md b/codes/news_rec_server/materials/README.md new file mode 100644 index 00000000..4b3cf10d --- /dev/null +++ b/codes/news_rec_server/materials/README.md @@ -0,0 +1,29 @@ +# 物料库 + +## news_scrapy/ + +scrapy项目结构的细节可以参考开源项目对应的文档 + +- 每天定时从新浪新闻上面爬取当天的新闻，并将新闻存入到mongodb数据库中 +- 爬取新闻是一个增量的过程，每天只爬取当天的新闻 +- 新闻爬取的时候需要提前之前物料池中的所有物料的标题给存下来用来去重 + + +TODO: +1. 启动crontab，定时爬取（联调的时候再使用） +2. 爬取新闻的数量，最后需要改大一点 + +## materials/ + +- 根据爬取的数据制作物料画像 +- 新闻物料去重逻辑的实现，需要依赖materials/中最新的物料画像 + +TODO: +1. 将前端展示数据根据news_id存储到redis中 + + +注意：Redis的value存储中文后，get之后显示16进制的字符串”\xe4\xb8\xad\xe5\x9b\xbd”，如何解决？ +启动redis-cli时，在其后面加上--raw即可，汉字即可显示正常, 如上面所示的内容 + +注意：zrange rec_list 0 9 返回的是排序之后的前10个元素，并且是按照value从小到大进行排序的 + diff --git a/codes/news_rec_server/materials/material_process/README.md b/codes/news_rec_server/materials/material_process/README.md new file mode 100644 index 00000000..3ec8b040 --- /dev/null +++ b/codes/news_rec_server/materials/material_process/README.md @@ -0,0 +1,5 @@ + +当前目录是用来处理数据的 + +1. 将爬取的新闻原始数据处理成画像数据，对应的脚本：update_news_protrait.py +2. 将处理好的特征画像中需要展示的数据放到redis中，对应的脚本：news_to_redis.py \ No newline at end of file diff --git a/codes/news_rec_server/materials/material_process/__pycache__/config.cpython-38.pyc b/codes/news_rec_server/materials/material_process/__pycache__/config.cpython-38.pyc new file mode 100644 index 00000000..24d79d48 Binary files /dev/null and b/codes/news_rec_server/materials/material_process/__pycache__/config.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/material_process/__pycache__/news_protrait.cpython-38.pyc b/codes/news_rec_server/materials/material_process/__pycache__/news_protrait.cpython-38.pyc new file mode 100644 index 00000000..bacb7975 Binary files /dev/null and b/codes/news_rec_server/materials/material_process/__pycache__/news_protrait.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/material_process/__pycache__/news_to_redis.cpython-38.pyc b/codes/news_rec_server/materials/material_process/__pycache__/news_to_redis.cpython-38.pyc new file mode 100644 index 00000000..d46f8db5 Binary files /dev/null and b/codes/news_rec_server/materials/material_process/__pycache__/news_to_redis.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-38.pyc b/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-38.pyc new file mode 100644 index 00000000..72b85daa Binary files /dev/null and b/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-39.pyc b/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-39.pyc new file mode 100644 index 00000000..5a20809b Binary files /dev/null and b/codes/news_rec_server/materials/material_process/__pycache__/utils.cpython-39.pyc differ diff --git a/codes/news_rec_server/materials/material_process/log_process.py b/codes/news_rec_server/materials/material_process/log_process.py new file mode 100644 index 00000000..a0748571 --- /dev/null +++ b/codes/news_rec_server/materials/material_process/log_process.py @@ -0,0 +1,32 @@ +import sys +sys.path.append('../../') + +from dao.mysql_server import MysqlServer +from dao.entity.user_read import UserRead +from dao.entity.logitem import LogItem +from controller.user_action_controller import UserAction + + +class LogProcess(object): + def __init__(self): + # 建立获取日志信息表的会话 + self.user_log_sql_session = MysqlServer().get_loginfo_session() + + def readlog_to_mysql(self): + # 拿到当天的用户阅读日志表 + logtables = self.user_log_sql_session.query(LogItem).filter_by(actiontype="read").all() + + # 遍历当天的用户日志表 + # 对于每条数据， 建立UserRead类， 通过UserAction保存到数据库里面去 + for log in logtables: + #print(log.userid, log.newsid, log.actiontype, log.actiontime) + user_read = UserRead() + user_read.new(log.userid, log.newsid, log.actiontime) + UserAction().save_one_action(user_read) + +if __name__ == "__main__": + LogProcess().readlog_to_mysql() + + + + diff --git a/codes/news_rec_server/materials/material_process/news_protrait.py b/codes/news_rec_server/materials/material_process/news_protrait.py new file mode 100644 index 00000000..f476b214 --- /dev/null +++ b/codes/news_rec_server/materials/material_process/news_protrait.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +from re import S +import sys +import json +sys.path.append("../") +from material_process.utils import get_key_words +from dao.mongo_server import MongoServer +from dao.redis_server import RedisServer + +""" +新闻画像中包含的字段： +0. news_id 新闻的id +1. title 标题 +2. raw_key_words (爬下来的关键词，可能有缺失) +3. manual_key_words (根据内容生成的关键词) +4. ctime 时间 +5. content 新闻具体内容 +6. cate 新闻类别 +7. likes 新闻点赞数量 +8. collections 新闻收藏数量 +9. read_nums 阅读次数 +10. url 新闻原始链接 +""" + +class NewsProtraitServer: + def __init__(self): + """初始化相关参数 + """ + self.mongo_server = MongoServer() + self.sina_collection = self.mongo_server.get_sina_news_collection() + self.material_collection = self.mongo_server.get_feature_protrail_collection() + self.redis_mongo_collection = self.mongo_server.get_redis_mongo_collection() + + self.news_dynamic_feature_redis = RedisServer().get_dynamic_news_info_redis() + + def _find_by_title(self, collection, title): + """从数据库中查找是否有相同标题的新闻数据 + 数据库存在当前标题的数据返回True, 反之返回Flase + """ + # find方法，返回的是一个迭代器 + find_res = collection.find({"title": title}) + if len(list(find_res)) != 0: + return True + return False + + def _generate_feature_protrail_item(self, item): + """生成特征画像数据，返回一个新的字典 + """ + news_item = dict() + news_item['news_id'] = item['news_id'] + news_item['title'] = item['title'] + # 从新闻内容中提取的关键词没有原始新闻爬取时的关键词准确，所以手动提取的关键词 + # 只是作为一个补充，当原始新闻中没有提供关键词的时候可以使用 + news_item['raw_key_words'] = item['raw_key_words'] + key_words_list = get_key_words(item['content']) + news_item['manual_key_words'] = ",".join(key_words_list) + news_item['ctime'] = item['ctime'] + news_item['content'] = item['content'] + news_item['cate'] = item['cate'] + news_item['url'] = item['url'] + news_item['likes'] = 0 + news_item['collections'] = 0 + news_item['read_num'] = 0 + news_item['hot_value'] = 1000 # 初始化一个比较大的热度值，会随着时间进行衰减 + + return news_item + + def update_new_items(self): + """将今天爬取的数据构造画像存入画像数据库中 + """ + # 遍历今天爬取的所有数据 + for item in self.sina_collection.find(): + # 根据标题进行去重 + if self._find_by_title(self.material_collection, item["title"]): + continue + + news_item = self._generate_feature_protrail_item(item) + + # 插入物料池 + self.material_collection.insert_one(news_item) + + print("run update_new_items success.") + + def update_redis_mongo_protrail_data(self): + """每天都需要将新闻详情更新到redis中，并且将前一天的redis数据删掉 + """ + # 每天先删除前一天的redis展示数据，然后再重新写入 + self.redis_mongo_collection.drop() + print("delete RedisProtrail ...") + # 遍历特征库 + for item in self.material_collection.find(): + news_item = dict() + news_item['news_id'] = item['news_id'] + news_item['title'] = item['title'] + news_item['ctime'] = item['ctime'] + news_item['content'] = item['content'] + news_item['cate'] = item['cate'] + news_item['url'] = item['url'] + news_item['likes'] = item['likes'] + news_item['collections'] = item['collections'] + news_item['read_num'] = item['read_num'] + + self.redis_mongo_collection.insert_one(news_item) + print("run update_redis_mongo_protrail_data success.") + + def update_dynamic_feature_protrail(self): + """用redis的动态画像更新mongodb的画像 + """ + # 遍历redis的动态画像，将mongodb中对应的动态画像更新 + news_list = self.news_dynamic_feature_redis.keys() + for news_key in news_list: + news_dynamic_info_str = self.news_dynamic_feature_redis.get(news_key) + news_dynamic_info_str = news_dynamic_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + news_dynamic_info_dict = json.loads(news_dynamic_info_str) + + # 查询mongodb中对应的数据，并将对应的画像进行修改 + news_id = news_key.split(":")[1] + mongo_info = self.material_collection.find_one({"news_id": news_id}) + new_mongo_info = mongo_info.copy() + new_mongo_info['likes'] = news_dynamic_info_dict["likes"] + new_mongo_info['collections'] = news_dynamic_info_dict["collections"] + new_mongo_info['read_num'] = news_dynamic_info_dict["read_num"] + + self.material_collection.replace_one(mongo_info, new_mongo_info, upsert=True) # upsert为True的话，没有就插入 + print("update_dynamic_feature_protrail success.") + + + +if __name__ == "__main__": + # TODO 需要放到 其他逻辑中，将物料这块的逻辑打通 + news_protrait = NewsProtraitServer() + # news_protrait.update_new_items() + news_protrait.update_redis_mongo_protrail_data() + # news_protrait.update_dynamic_feature_protrail() diff --git a/codes/news_rec_server/materials/material_process/news_to_redis.py b/codes/news_rec_server/materials/material_process/news_to_redis.py new file mode 100644 index 00000000..078c297c --- /dev/null +++ b/codes/news_rec_server/materials/material_process/news_to_redis.py @@ -0,0 +1,83 @@ +import sys +sys.path.append("../../") +from dao.mongo_server import MongoServer +from dao.redis_server import RedisServer + + +class NewsRedisServer(object): + def __init__(self): + self.rec_list_redis = RedisServer().get_reclist_redis() + self.static_news_info_redis = RedisServer().get_static_news_info_redis() + self.dynamic_news_info_redis = RedisServer().get_dynamic_news_info_redis() + + self.redis_mongo_collection = MongoServer().get_redis_mongo_collection() + + # 删除前一天redis中的内容 + self._flush_redis_db() + + def _flush_redis_db(self): + """每天都需要删除redis中的内容，更新当天新的内容上去 + """ + try: + self.rec_list_redis.flushall() + except Exception: + print("flush redis fail ... ") + + def _get_news_id_list(self): + """获取物料库中所有的新闻id + """ + # 获取所有数据的news_id, + # 暴力获取，直接遍历整个数据库，得到所有新闻的id + # TODO 应该存在优化方法可以通过查询的方式只返回new_id字段 + news_id_list = [] + for item in self.redis_mongo_collection.find(): + news_id_list.append(item["news_id"]) + return news_id_list + + def _set_info_to_redis(self, redisdb, content): + """将content添加到指定redis + """ + try: + redisdb.set(*content) + except Exception: + print("set content fail".format(content)) + + def news_detail_to_redis(self): + """将需要展示的画像内容存储到redis + 静态不变的特征存到static_news_info_db_num + 动态会发生改变的特征存到dynamic_news_info_db_num + """ + news_id_list = self._get_news_id_list() + + for news_id in news_id_list: + news_item_dict = self.redis_mongo_collection.find_one({"news_id": news_id}) # 返回的是一个列表里面套了一个字典 + news_item_dict.pop("_id") + + # 分离动态属性和静态属性 + static_news_info_dict = dict() + static_news_info_dict['news_id'] = news_item_dict['news_id'] + static_news_info_dict['title'] = news_item_dict['title'] + static_news_info_dict['ctime'] = news_item_dict['ctime'] + static_news_info_dict['content'] = news_item_dict['content'] + static_news_info_dict['cate'] = news_item_dict['cate'] + static_news_info_dict['url'] = news_item_dict['url'] + static_content_tuple = "static_news_detail:" + str(news_id), str(static_news_info_dict) + self._set_info_to_redis(self.static_news_info_redis, static_content_tuple) + + dynamic_news_info_dict = dict() + dynamic_news_info_dict['likes'] = news_item_dict['likes'] + dynamic_news_info_dict['collections'] = news_item_dict['collections'] + dynamic_news_info_dict['read_num'] = news_item_dict['read_num'] + dynamic_content_tuple = "dynamic_news_detail:" + str(news_id), str(dynamic_news_info_dict) + self._set_info_to_redis(self.dynamic_news_info_redis, dynamic_content_tuple) + + print("news detail info are saved in redis db.") + + +if __name__ == "__main__": + # 每次创建这个对象的时候都会把数据库中之前的内容删除 + news_redis_server = NewsRedisServer() + # 将最新的前端展示的画像传到redis + news_redis_server.news_detail_to_redis() + + \ No newline at end of file diff --git a/codes/news_rec_server/materials/material_process/utils.py b/codes/news_rec_server/materials/material_process/utils.py new file mode 100644 index 00000000..82cba3f8 --- /dev/null +++ b/codes/news_rec_server/materials/material_process/utils.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +这里主要实现一些工具类的函数 +""" +import re +import sys +sys.path.append("../../") +import jieba +import jieba.analyse + + +def get_key_words(words_str): + """提取中文中的关键词 + """ + # 字符串进行清洗 + # 去除一些符号 + words_str.replace('\n', '').replace('\u3000', '').replace('\u00A0', '') + # 去除数字，特殊字符 + words_str = re.sub('[a-zA-Z0-9.。:：,，]', '', words_str) + # 切词 + words_list = jieba.cut(words_str) + + # 加载停用词 + stopword_set = set() + # TODO 改成变量而不是写死 + with open('/home/recsys/news_rec_server/conf/stop_words.txt', encoding="utf-8") as f: + line = f.readline().rstrip() + stopword_set.add(line) + + # 去除停用词 + new_words_list = [] + for word in words_list: + if word in stopword_set: + continue + new_words_list.append(word) + + new_words_str = " ".join(new_words_list) + + # 提取关键词 + # 默认是TF-IDF + key_words_list_tfidf = jieba.analyse.extract_tags(new_words_str, topK=10, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v')) + key_words_list_textrank = jieba.analyse.textrank(new_words_str, topK=10, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v')) + + # print("key_words_list_tfidf", key_words_list_tfidf) + # print("key_words_list_textrank", key_words_list_textrank) + + tfidf_textrank_list = list(set(key_words_list_tfidf) & set(key_words_list_textrank))[:3] + + # print(tfidf_textrank_list) + return tfidf_textrank_list diff --git a/codes/news_rec_server/materials/news_scrapy/monitor_news.py b/codes/news_rec_server/materials/news_scrapy/monitor_news.py new file mode 100644 index 00000000..152aef7c --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/monitor_news.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +import sys, time +import pymongo +from sinanews.settings import MONGO_HOST, MONGO_PORT, SINA_DB_NAME, COLLECTION_NAME_PRFIX + +if __name__ == "__main__": + news_num = int(sys.argv[1]) + time_str = time.strftime("%Y%m%d", time.localtime()) + + # 实际的collection_name + collection_name = COLLECTION_NAME_PRFIX + "_" + time_str + + # 链接数据库 + client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT) + db = client[SINA_DB_NAME] + collection = db[collection_name] + + # 查找当前集合中所有文档的数量 + cur_news_num = collection.count() + + if (cur_news_num < news_num): + print("the news nums of {}_{} collection is {} and less then {}.".\ + format(COLLECTION_NAME_PRFIX, time_str, cur_news_num, news_num)) \ No newline at end of file diff --git a/codes/news_rec_server/materials/news_scrapy/scrapy.cfg b/codes/news_rec_server/materials/news_scrapy/scrapy.cfg new file mode 100644 index 00000000..ac86f3dc --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = sinanews.settings + +[deploy] +#url = http://localhost:6800/ +project = sinanews diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/__init__.py b/codes/news_rec_server/materials/news_scrapy/sinanews/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/__init__.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 00000000..0d46df0a Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/__init__.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/items.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/items.cpython-38.pyc new file mode 100644 index 00000000..553e0640 Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/items.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/pipelines.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/pipelines.cpython-38.pyc new file mode 100644 index 00000000..d3a2bae2 Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/pipelines.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/settings.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/settings.cpython-38.pyc new file mode 100644 index 00000000..a08a53d4 Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/__pycache__/settings.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/items.py b/codes/news_rec_server/materials/news_scrapy/sinanews/items.py new file mode 100644 index 00000000..f96cff60 --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/items.py @@ -0,0 +1,19 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy +from scrapy import Item, Field + +# 定义新闻数据的字段 +class SinanewsItem(scrapy.Item): + """数据格式化，数据不同字段的定义 + """ + title = Field() # 新闻标题 + ctime = Field() # 新闻发布时间 + url = Field() # 新闻原始url + raw_key_words = Field() # 新闻关键词（爬取的关键词） + content = Field() # 新闻的具体内容 + cate = Field() # 新闻类别 + news_id = Field() # 新闻id \ No newline at end of file diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/middlewares.py b/codes/news_rec_server/materials/news_scrapy/sinanews/middlewares.py new file mode 100644 index 00000000..be17b83b --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/middlewares.py @@ -0,0 +1,103 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class SinanewsSpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class SinanewsDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/pipelines.py b/codes/news_rec_server/materials/news_scrapy/sinanews/pipelines.py new file mode 100644 index 00000000..67e99c8d --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/pipelines.py @@ -0,0 +1,74 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +import time +import datetime +import pymongo +from pymongo.errors import DuplicateKeyError +from sinanews.items import SinanewsItem +from itemadapter import ItemAdapter + + +# 新闻item持久化 +class SinanewsPipeline: + """数据持久化：将数据存放到mongodb中 + """ + def __init__(self, host, port, db_name, collection_name): + self.host = host + self.port = port + self.db_name = db_name + self.collection_name = collection_name + + @classmethod + def from_crawler(cls, crawler): + """自带的方法，这个方法可以重新返回一个新的pipline对象，并且可以调用配置文件中的参数 + """ + return cls( + host = crawler.settings.get("MONGO_HOST"), + port = crawler.settings.get("MONGO_PORT"), + db_name = crawler.settings.get("SINA_DB_NAME"), + # mongodb中数据的集合按照日期存储 + collection_name = crawler.settings.get("COLLECTION_NAME_PRFIX") + \ + "_" + time.strftime("%Y%m%d", time.localtime()) + ) + + def open_spider(self, spider): + """开始爬虫的操作，主要就是链接数据库及对应的集合 + """ + self.client = pymongo.MongoClient(self.host, self.port) + self.db = self.client[self.db_name] + self.collection = self.db[self.collection_name] + + def close_spider(self, spider): + """关闭爬虫操作的时候，需要将数据库断开 + """ + self.client.close() + + def process_item(self, item, spider): + """处理每一条数据，注意这里需要将item返回 + 注意：判断新闻是否是今天的，每天只保存当天产出的新闻，这样可以增量的添加新的新闻数据源 + """ + if isinstance(item, SinanewsItem): + try: + # TODO 物料去重逻辑，根据title进行去重，去重逻辑在画像处理环节实现 + + # 当前新闻的时间戳 + cur_time = int(time.mktime(time.strptime(item['ctime'], '%Y-%m-%d %H:%M'))) + + # str_today = str(datetime.date.today()) + # 每次爬取新闻的时候是爬取前一天的 + yestoday = str(datetime.date.today() - datetime.timedelta(days=1)) + min_time = int(time.mktime(time.strptime(yestoday + " 00:00:00", '%Y-%m-%d %H:%M:%S'))) + max_time = int(time.mktime(time.strptime(yestoday + " 23:59:59", '%Y-%m-%d %H:%M:%S'))) + if cur_time > min_time and cur_time <= max_time: + self.collection.insert_one(dict(item)) + except DuplicateKeyError: + """ + 说明有重复 + """ + pass + return item diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/run.py b/codes/news_rec_server/materials/news_scrapy/sinanews/run.py new file mode 100644 index 00000000..ff1303b5 --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/run.py @@ -0,0 +1,4 @@ +from scrapy import cmdline + +# 注意这里pages需要进行调整 +cmdline.execute('scrapy crawl sina_spider --nolog -a pages=50'.split()) \ No newline at end of file diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/settings.py b/codes/news_rec_server/materials/news_scrapy/sinanews/settings.py new file mode 100644 index 00000000..c31a6d75 --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/settings.py @@ -0,0 +1,101 @@ +# Scrapy settings for sinanews project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from typing import Collection +import sys +sys.path.append("/home/recsys/news_rec_server") +from conf.dao_config import mongo_hostname, mongo_port, sina_db_name, sina_collection_name_prefix + +BOT_NAME = 'sinanews' + +SPIDER_MODULES = ['sinanews.spiders'] +NEWSPIDER_MODULE = 'sinanews.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'sinanews (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +# DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'sinanews.middlewares.SinanewsSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'sinanews.middlewares.SinanewsDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +# 如果需要使用itempipline来存储item的话需要将这段注释打开 +ITEM_PIPELINES = { + 'sinanews.pipelines.SinanewsPipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + +MONGO_HOST = mongo_hostname +MONGO_PORT = mongo_port +SINA_DB_NAME = sina_db_name +COLLECTION_NAME_PRFIX = sina_collection_name_prefix + + diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__init__.py b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__init__.py new file mode 100644 index 00000000..ebd689ac --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/__init__.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 00000000..e173ddbc Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/__init__.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/sina.cpython-38.pyc b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/sina.cpython-38.pyc new file mode 100644 index 00000000..eb177f0e Binary files /dev/null and b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/__pycache__/sina.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/sina.py b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/sina.py new file mode 100644 index 00000000..8c3fa227 --- /dev/null +++ b/codes/news_rec_server/materials/news_scrapy/sinanews/spiders/sina.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +import re +import uuid +import json +import random +import scrapy +from scrapy import Request +from ..items import SinanewsItem +from datetime import datetime + + +class SinaSpider(scrapy.Spider): + # spider的名字 + name = 'sina_spider' + + def __init__(self, pages=None): + super(SinaSpider).__init__() + + self.total_pages = int(pages) + # base_url 对应的是新浪新闻的简洁版页面，方便爬虫，并且不同类别的新闻也很好区分 + self.base_url = 'https://feed.mix.sina.com.cn/api/roll/get?pageid=153&lid={}&k=&num=50&page={}&r={}' + # lid和分类映射字典 + self.cate_dict = { + "2510": "国内", + "2511": "国际", + "2669": "社会", + "2512": "体育", + "2513": "娱乐", + "2514": "军事", + "2515": "科技", + "2516": "财经", + "2517": "股市", + "2518": "美股" + } + + def start_requests(self): + """返回一个Request迭代器 + """ + # 遍历所有类型的论文 + for cate_id in self.cate_dict.keys(): + for page in range(1, self.total_pages + 1): + lid = cate_id + # 这里就是一个随机数，具体含义不是很清楚 + r = random.random() + # cb_kwargs 是用来往解析函数parse中传递参数的 + yield Request(self.base_url.format(lid, page, r), callback=self.parse, cb_kwargs={"cate_id": lid}) + + def parse(self, response, cate_id): + """解析网页内容，并提取网页中需要的内容 + """ + + json_result = json.loads(response.text) # 将请求回来的页面解析成json + # 提取json中我们想要的字段 + # json使用get方法比直接通过字典的形式获取数据更方便，因为不需要处理异常 + data_list = json_result.get('result').get('data') + for data in data_list: + item = SinanewsItem() + + # 给当前文章生成一个唯一的id + item['news_id'] = str(uuid.uuid4()) # 通过随机数来生成UUID. 使用的是伪随机数有一定的重复概率. + item['cate'] = self.cate_dict[cate_id] + item['title'] = data.get('title') + item['url'] = data.get('url') + item['raw_key_words'] = data.get('keywords') + + ctime = datetime.fromtimestamp(int(data.get('ctime'))) + ctime = datetime.strftime(ctime, '%Y-%m-%d %H:%M') + + # 保留格式化之后的时间戳 + item['ctime'] = ctime + + # meta参数传入的是一个字典，在下一层可以将当前层的item进行复制 + yield Request(url=item['url'], callback=self.parse_content, meta={'item': item}) + + def parse_content(self, response): + """解析文章内容 + """ + item = response.meta['item'] + content = ''.join(response.xpath('//*[@id="artibody" or @id="article"]//p/text()').extract()) + content = re.sub(r'\u3000', '', content) + content = re.sub(r'[ \xa0?]+', ' ', content) + content = re.sub(r'\s*\n\s*', '\n', content) + content = re.sub(r'\s*(\s)', r'\1', content) + content = ''.join([x.strip() for x in content]) + item['content'] = content + yield item + diff --git a/codes/news_rec_server/materials/process_material.py b/codes/news_rec_server/materials/process_material.py new file mode 100644 index 00000000..45f50a45 --- /dev/null +++ b/codes/news_rec_server/materials/process_material.py @@ -0,0 +1,20 @@ +from material_process.news_protrait import NewsProtraitServer +from material_process.news_to_redis import NewsRedisServer + + +def process_material(): + """物料处理函数 + """ + # 画像处理 + protrail_server = NewsProtraitServer() + # 处理最新爬取新闻的画像，存入特征库 + protrail_server.update_new_items() + # 更新新闻动态画像, 需要在redis数据库内容清空之前执行 + protrail_server.update_dynamic_feature_protrail() + # 生成前端展示的新闻画像，并在mongodb中备份一份 + protrail_server.update_redis_mongo_protrail_data() + + +if __name__ == "__main__": + process_material() + diff --git a/codes/news_rec_server/materials/process_user.py b/codes/news_rec_server/materials/process_user.py new file mode 100644 index 00000000..88cd0aff --- /dev/null +++ b/codes/news_rec_server/materials/process_user.py @@ -0,0 +1,25 @@ +from user_process.user_to_mysql import UserMysqlServer +from user_process.user_protrail import UserProtrail + +""" +1. 将用户的曝光数据从redis落到mysql中。 +2. 更新用户画像 +""" + + +def process_users(): + """将用户数据落 Mysql + """ + # 用户mysql存储 + user_mysql_server = UserMysqlServer() + # 用户曝光数据落mysql + user_mysql_server.user_exposure_to_mysql() + + # 更新用户画像 + user_protrail = UserProtrail() + user_protrail.update_user_protrail_from_register_table() + + +if __name__ == "__main__": + process_users() + diff --git a/codes/news_rec_server/materials/update_redis.py b/codes/news_rec_server/materials/update_redis.py new file mode 100644 index 00000000..53299fa8 --- /dev/null +++ b/codes/news_rec_server/materials/update_redis.py @@ -0,0 +1,16 @@ +from material_process.news_protrait import NewsProtraitServer +from material_process.news_to_redis import NewsRedisServer + + +def update(): + """物料处理函数 + """ + # 新闻数据写入redis, 注意这里处理redis数据的时候是会将前一天的数据全部清空 + news_redis_server = NewsRedisServer() + # 将最新的前端展示的画像传到redis + news_redis_server.news_detail_to_redis() + + +if __name__ == "__main__": + update() + diff --git a/codes/news_rec_server/materials/user_process/__pycache__/user_protrail.cpython-38.pyc b/codes/news_rec_server/materials/user_process/__pycache__/user_protrail.cpython-38.pyc new file mode 100644 index 00000000..6729c841 Binary files /dev/null and b/codes/news_rec_server/materials/user_process/__pycache__/user_protrail.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/user_process/__pycache__/user_to_mysql.cpython-38.pyc b/codes/news_rec_server/materials/user_process/__pycache__/user_to_mysql.cpython-38.pyc new file mode 100644 index 00000000..28245c4b Binary files /dev/null and b/codes/news_rec_server/materials/user_process/__pycache__/user_to_mysql.cpython-38.pyc differ diff --git a/codes/news_rec_server/materials/user_process/user_protrail.py b/codes/news_rec_server/materials/user_process/user_protrail.py new file mode 100644 index 00000000..da75aa97 --- /dev/null +++ b/codes/news_rec_server/materials/user_process/user_protrail.py @@ -0,0 +1,142 @@ +import sys +import datetime +from collections import Counter, defaultdict + +from sqlalchemy.sql.expression import table +sys.path.append("../../") +from dao.mongo_server import MongoServer +from dao.mysql_server import MysqlServer +from dao.entity.register_user import RegisterUser +from dao.entity.user_read import UserRead +from dao.entity.user_likes import UserLikes +from dao.entity.user_collections import UserCollections + + +class UserProtrail(object): + def __init__(self): + self.user_protrail_collection = MongoServer().get_user_protrail_collection() + self.material_collection = MongoServer().get_feature_protrail_collection() + self.register_user_sess = MysqlServer().get_register_user_session() + self.user_collection_sess = MysqlServer().get_user_collection_session() + self.user_like_sess = MysqlServer().get_user_like_session() + self.user_read_sess = MysqlServer().get_user_read_session() + + def _user_info_to_dict(self, user): + """将mysql查询出来的结果转换成字典存储 + """ + info_dict = dict() + + # 基本属性特征 + info_dict["userid"] = user.userid + info_dict["username"] = user.username + info_dict["passwd"] = user.passwd + info_dict["gender"] = user.gender + info_dict["age"] = user.age + info_dict["city"] = user.city + + # 兴趣爱好 + behaviors=["like","collection"] + time_range = 15 + _, feature_dict = self.get_statistical_feature_from_history_behavior(user.userid,time_range,behavior_types=behaviors) + for type in feature_dict.keys(): + if feature_dict[type]: + info_dict["{}_{}_intr_cate".format(type,time_range)] = feature_dict[type]["intr_cate"] # 历史喜欢最多的Top3的新闻类别 + info_dict["{}_{}_intr_key_words".format(type,time_range)] = feature_dict[type]["intr_key_words"] # 历史喜欢新闻的Top3的关键词 + info_dict["{}_{}_avg_hot_value".format(type,time_range)] = feature_dict[type]["avg_hot_value"] # 用户喜欢新闻的平均热度 + info_dict["{}_{}_news_num".format(type,time_range)] = feature_dict[type]["news_num"] # 用户15天内喜欢的新闻数量 + else: + info_dict["{}_{}_intr_cate".format(type,time_range)] = "" # 历史喜欢最多的Top3的新闻类别 + info_dict["{}_{}_intr_key_words".format(type,time_range)] = "" # 历史喜欢新闻的Top3的关键词 + info_dict["{}_{}_avg_hot_value".format(type,time_range)] = 0 # 用户喜欢新闻的平均热度 + info_dict["{}_{}_news_num".format(type,time_range)] = 0 # 用户15天内喜欢的新闻数量 + + return info_dict + + def update_user_protrail_from_register_table(self): + """每天都需要将当天注册的用户添加到用户画像池中 + """ + # 遍历注册用户表 + for user in self.register_user_sess.query(RegisterUser).all(): + user_info_dict = self._user_info_to_dict(user) + old_user_protrail_dict = self.user_protrail_collection.find_one({"username": user.username}) + if old_user_protrail_dict is None: + self.user_protrail_collection.insert_one(user_info_dict) + else: + # 使用参数upsert设置为true对于没有的会创建一个 + # replace_one 如果遇到相同的_id 就会更新 + self.user_protrail_collection.replace_one(old_user_protrail_dict, user_info_dict, upsert=True) + + + def get_statistical_feature_from_history_behavior(self, user_id, time_range, behavior_types): + """获取用户历史行为的统计特征 ["read","like","collection"] """ + fail_type = [] + sess, table_obj, history = None, None, None + feature_dict = defaultdict(dict) + + end = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + start = (datetime.datetime.now()+datetime.timedelta(days=-time_range)).strftime("%Y-%m-%d %H:%M:%S") + + for type in behavior_types: + if type == "read": + sess = getattr(self,"user_{}_sess".format(type)) + table_obj = UserRead + elif type == "like": + sess = getattr(self,"user_{}_sess".format(type)) + table_obj = UserLikes + elif type == "collection": + sess = getattr(self,"user_{}_sess".format(type)) + table_obj = UserCollections + try: + history = sess.query(table_obj).filter(table_obj.userid==user_id).filter(table_obj.curtime>=start).filter(table_obj.curtime<=end).all() + except Exception as e: + print(str(e)) + fail_type.append(type) + continue + + feature_dict[type] = self._gen_statistical_feature(history) + + return fail_type, feature_dict + + def _gen_statistical_feature(self,history): + """""" + # 为history 获取特征 + if not len(history): return None + history_new_id = [] + history_hot_value = [] + history_new_cate = [] + history_key_word = [] + for h in history: + news_id = h.newid + newsquery = {"news_id":news_id} + result = self.material_collection.find_one(newsquery) + history_new_id.append(result["news_id"]) + history_hot_value.append(result["hot_value"]) + history_new_cate.append(result["cate"]) + history_key_word += result["manual_key_words"].split(",") + + feature_dict = dict() + # 计算平均热度 + feature_dict["avg_hot_value"] = 0 if sum(history_hot_value) < 0.001 else sum(history_hot_value) / len(history_hot_value) + + # 计算Top3的类别 + cate_dict = Counter(history_new_cate) + cate_list= sorted(cate_dict.items(),key = lambda d: d[1], reverse=True) + cate_str = ",".join([item[0] for item in cate_list[:3]] if len(cate_list)>=3 else [item[0] for item in cate_list] ) + feature_dict["intr_cate"] = cate_str + + # 计算Top3的关键词 + word_dict = Counter(history_key_word) + word_list= sorted(word_dict.items(),key = lambda d: d[1], reverse=True) + # TODO 关键字属于长尾 如果关键字的次数都是一次 该怎么去前3 + word_str = ",".join([item[0] for item in word_list[:3]] if len(cate_list)>=3 else [item[0] for item in word_list] ) + feature_dict["intr_key_words"] = word_str + + # 新闻数目 + feature_dict["news_num"] = len(history_new_id) + + return feature_dict + + +if __name__ == "__main__": + user_protrail = UserProtrail().update_user_protrail_from_register_table() + # user_protrail = UserProtrail().get_statistical_feature_from_history_behavior(user_id="4563323680741920769") diff --git a/codes/news_rec_server/materials/user_process/user_to_mysql.py b/codes/news_rec_server/materials/user_process/user_to_mysql.py new file mode 100644 index 00000000..fd9c2483 --- /dev/null +++ b/codes/news_rec_server/materials/user_process/user_to_mysql.py @@ -0,0 +1,43 @@ +import sys +sys.path.append("./../") +from dao.redis_server import RedisServer +from dao.mysql_server import MysqlServer +from dao.entity.user_exposure import UserExposure + + +class UserMysqlServer(object): + def __init__(self): + self.user_exposure_redis = RedisServer().get_exposure_redis() + self.user_exposure_sql_session = MysqlServer().get_user_exposure_session() + + def user_exposure_to_mysql(self): + exposure = UserExposure() # 为了通过__init__()函数构建表 + vals = [] + keys = self.user_exposure_redis.keys() + for key in keys: + news_list = self.user_exposure_redis.smembers(key) + user_id = key.split(":")[1] + val = self._transfor_json_for_user(user_id,news_list) + vals +=val + + self.user_exposure_sql_session.bulk_insert_mappings(UserExposure,vals) + self.user_exposure_sql_session.commit() + + def _transfor_json_for_user(self,user_id,news_list): + """针对每个用户转换成批量存储的形式 + """ + # 对用户的每一个曝光进行存储 + vals = [] + for item in news_list: + item = item.split(":") + vals.append({ + "userid":user_id, + "newid":item[0], + "curtime":item[1]}) + return vals + + +if __name__ == "__main__": + user_mysql_server = UserMysqlServer() + user_mysql_server.user_exposure_to_mysql() + \ No newline at end of file diff --git a/codes/news_rec_server/recprocess/README.md b/codes/news_rec_server/recprocess/README.md new file mode 100644 index 00000000..ed8d55cd --- /dev/null +++ b/codes/news_rec_server/recprocess/README.md @@ -0,0 +1,4 @@ + +当前文件主要是存放个性化推荐流程相关的内容 + +recsys.py 就是一个单例类，为后端提供推荐服务 \ No newline at end of file diff --git a/codes/news_rec_server/recprocess/__pycache__/online.cpython-38.pyc b/codes/news_rec_server/recprocess/__pycache__/online.cpython-38.pyc new file mode 100644 index 00000000..8a6d8fc6 Binary files /dev/null and b/codes/news_rec_server/recprocess/__pycache__/online.cpython-38.pyc differ diff --git a/codes/news_rec_server/recprocess/__pycache__/recsys.cpython-38.pyc b/codes/news_rec_server/recprocess/__pycache__/recsys.cpython-38.pyc new file mode 100644 index 00000000..5eac58f7 Binary files /dev/null and b/codes/news_rec_server/recprocess/__pycache__/recsys.cpython-38.pyc differ diff --git a/codes/news_rec_server/recprocess/cold_start/cold_start.py b/codes/news_rec_server/recprocess/cold_start/cold_start.py new file mode 100644 index 00000000..d9573d5d --- /dev/null +++ b/codes/news_rec_server/recprocess/cold_start/cold_start.py @@ -0,0 +1,62 @@ +import sys +sys.path.append('../../') + +from dao.mongo_server import MongoServer +from dao.redis_server import RedisServer +from datetime import datetime + + +# 这里设计冷启动规则 +# 针对每个用户，构造出三部分最终将计算好的物料热度，对物料进行排序 + +class HotRecall(object): + def __init__(self): + self.feature_protrail_collection = MongoServer().get_feature_protrail_collection() + self.reclist_redis = RedisServer().get_reclist_redis() + + def get_hot_rec_list(self): + """获取物料的点赞，收藏和创建时间等信息，计算热度并生成热度推荐列表存入redis + """ + # 遍历物料池里面的所有文章 + for item in self.feature_protrail_collection.find(): + news_id = item['news_id'] + news_cate = item['cate'] + news_ctime = item['ctime'] + news_likes_num = item['likes'] + news_collections_num = item['collections'] + news_read_num = item['read_num'] + news_hot_value = item['hot_value'] + + #print(news_id, news_cate, news_ctime, news_likes_num, news_collections_num, news_read_num, news_hot_value) + + # 时间转换与计算时间差 前提要保证当前时间大于新闻创建时间，目前没有捕捉异常 + news_ctime_standard = datetime.strptime(news_ctime, "%Y-%m-%d %H:%M") + cur_time_standard = datetime.now() + time_day_diff = (cur_time_standard - news_ctime_standard).days + time_hour_diff = (cur_time_standard - news_ctime_standard).seconds / 3600 + + # 只要最近3天的内容 + if time_day_diff > 3: + continue + + # 计算热度分，这里使用魔方秀热度公式， 可以进行调整, read_num 上一次的 hot_value 上一次的hot_value用加？ 因为like_num这些也都是累加上来的， 所以这里计算的并不是增值，而是实时热度吧 + # news_hot_value = (news_likes_num * 6 + news_collections_num * 3 + news_read_num * 1) * 10 / (time_hour_diff+1)**1.2 + # 72 表示的是3天， + news_hot_value = (news_likes_num * 0.6 + news_collections_num * 0.3 + news_read_num * 0.1) * 10 / (1 + time_hour_diff / 72) + + #print(news_likes_num, news_collections_num, time_hour_diff) + + # 更新物料池的文章hot_value + item['hot_value'] = news_hot_value + self.feature_protrail_collection.update({'news_id':news_id}, item) + + #print("news_hot_value: ", news_hot_value) + + # 保存到redis中 + self.reclist_redis.zadd('hot_list', {'{}_{}'.format(news_cate, news_id): news_hot_value}, nx=True) + + + + + + diff --git a/codes/news_rec_server/recprocess/offline.py b/codes/news_rec_server/recprocess/offline.py new file mode 100644 index 00000000..0fec1e99 --- /dev/null +++ b/codes/news_rec_server/recprocess/offline.py @@ -0,0 +1,69 @@ + +import sys +sys.path.append("../") +from dao.mongo_server import MongoServer +from dao.redis_server import RedisServer +from recall.hot_recall import HotRecall +from datetime import datetime + +# 这个类是用来实现离线推荐流程的，给每个用户都存储一个倒排索引列表 +# 对于热门页的内容，初始化的时候每个用户都是一样的 +# 这个类的实例会在处理完物料之后，生成当前最新的用户推荐列表，热门列表，及冷启动内容模板 +class OfflineServer(object): + def __init__(self): + self.redis_mongo_collection = MongoServer().get_redis_mongo_collection() + self.reclist_redis = RedisServer().get_reclist_redis() + self.hot_recall = HotRecall() + + + def hot_list_to_redis(self): + """热门页面，初始化的时候每个用户都是同样的内容 + """ + self.hot_recall.get_hot_rec_list() + print("a hot rec list are saved into redis.....") + + def rec_list_to_redis(self): + """个性化推荐列表，千人千面 + """ + pass + + def cold_start_template_to_redis(self): + """冷启动列表模板 + """ + pass + + def _get_user_id_list(self): + """获取所有注册用户的id + """ + pass + + def _get_news_id_list(self): + """获取所有新闻id + """ + # 获取所有数据的news_id, + # 暴力获取，直接遍历整个数据库，得到所有新闻的id + # TODO 应该存在优化方法可以通过查询的方式只返回new_id字段 + news_id_list = [] + for item in self.redis_mongo_collection.find(): + news_id_list.append(item["news_id"]) + return news_id_list + + def test(self): + """给redis中加入一个倒排索引用于测试 + """ + news_id_list = self._get_news_id_list() + value_list = [i for i in range(len(news_id_list))] + + for news_id, val in zip(news_id_list, value_list): + # print({news_id: val}) + self.reclist_redis.zadd("rec_list", {news_id: val}, nx=True) + + print("a sorted news_ids are saved into redis.") + + +if __name__ == "__main__": + # 生成统一的，用于测试的倒排索引 + # TODO 待测试 + offline_server = OfflineServer().test() + offline_server = OfflineServer().hot_list_to_redis() + diff --git a/codes/news_rec_server/recprocess/online.py b/codes/news_rec_server/recprocess/online.py new file mode 100644 index 00000000..469d81f8 --- /dev/null +++ b/codes/news_rec_server/recprocess/online.py @@ -0,0 +1,262 @@ +import sys +sys.path.append("../../") +sys.path.append("../") +import json +import time +import threading + +from dao.redis_server import RedisServer + +from controller.user_action_controller import UserAction + +redis_server = RedisServer() + +class OnlineServer(object): + """单例模式推荐服务类 + """ + _instance_lock = threading.Lock() + + def __init__(self,): + self.reclist_redis_db = redis_server.get_reclist_redis() + self.static_news_info_redis_db = redis_server.get_static_news_info_redis() + self.dynamic_news_info_redis_db = redis_server.get_dynamic_news_info_redis() + self.exposure_redis_db = redis_server.get_exposure_redis() + + def __new__(cls, *args, **kwargs): + if not hasattr(OnlineServer, "_instance"): + with OnlineServer._instance_lock: + if not hasattr(OnlineServer, "_instance"): + OnlineServer._instance = object.__new__(cls) + return OnlineServer._instance + + def get_rec_list(self, user_id, page_id): + """给定页面的展示范围进行展示 + user_id 后面做个性化推荐的时候需要用到 + """ + # 根据page id计算需要获取redis中哪些范围的news_id, 假设每一页展示10个新闻 + s = (int(page_id) - 1) * 10 + e = s + 9 + + # 返回的是一个news_id列表 + news_id_list = self.reclist_redis_db.zrange("rec_list", start=s, end=e) + # print(news_id_list) + + # 根据news_id获取新闻的具体内容，并返回一个列表，列表中的元素是按照顺序展示的新闻信息字典 + news_info_list = [] + news_expose_list = [] + for news_id in news_id_list: + news_info_dict = self._get_news_simple(news_id) + # news_info_str = news_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + # news_info_dict = json.loads(news_info_str) + # 需要确认一下前端接收的json，key需要是单引号还是双引号 + news_info_list.append(news_info_dict) + news_expose_list.append(news_info_dict["news_id"]) # 记录在用户曝光表上[user_exposure] + + # UserAction().save_user_exposure(user_id,news_expose_list) # 曝光落表 + self._save_user_exposure(user_id,news_expose_list) + + return news_info_list + + def _get_news_simple(self, news_id): + """获取新闻的简略信息,用于刚进来时的展示 + """ + news_info_str = self.static_news_info_redis_db.get("static_news_detail:" + news_id) + news_info_str = news_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + news_info_str = json.loads(news_info_str) + + news_dynamic_info_str = self.dynamic_news_info_redis_db.get("dynamic_news_detail:" + news_id) + news_dynamic_info_str = news_dynamic_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + news_dynamic_info_str = json.loads(news_dynamic_info_str) + + simple = ["news_id","title","ctime","cate"] + for k in simple: + news_dynamic_info_str[k] = news_info_str[k] + + return news_dynamic_info_str + + def get_hot_list(self, user_id): + """热门页列表结果""" + hot_list_key_prefix = "user_id_hot_list:" + hot_list_user_key = hot_list_key_prefix + str(user_id) + + user_exposure_prefix = "user_exposure:" + user_exposure_key = user_exposure_prefix + str(user_id) + + # 当数据库中没有这个用户的数据，就从热门列表中拷贝一份 + if self.reclist_redis_db.exists(hot_list_user_key) == 0: # 存在返回1，不存在返回0 + print("copy a hot_list for {}".format(hot_list_user_key)) + # 给当前用户重新生成一个hot页推荐列表， 也就是把hot_list里面的列表复制一份给当前user， key换成user_id + self.reclist_redis_db.zunionstore(hot_list_user_key, ["hot_list"]) + + # 一页默认10个item, 但这里候选20条，因为有可能有的在推荐页曝光过 + article_num = 50 + + # 返回的是一个news_id列表 zrevrange排序分值从大到小 + candiate_id_list = self.reclist_redis_db.zrevrange(hot_list_user_key, 0, article_num-1) + + print("candiate_id_list", candiate_id_list) + + if len(candiate_id_list) > 0: + # 根据news_id获取新闻的具体内容，并返回一个列表，列表中的元素是按照顺序展示的新闻信息字典 + news_info_list = [] + selected_news = [] # 记录真正被选了的 + cou = 0 + + # 曝光列表 + print("self.reclist_redis_db.exists(key)",self.exposure_redis_db.exists(user_exposure_key)) + if self.exposure_redis_db.exists(user_exposure_key) > 0: + exposure_list = self.exposure_redis_db.smembers(user_exposure_key) + news_expose_list = set(map(lambda x: x.split(':')[0], exposure_list)) + else: + news_expose_list = set() + + for i in range(len(candiate_id_list)): + candiate = candiate_id_list[i] + news_id = candiate.split('_')[1] + + # 去重曝光过的，包括在推荐页以及hot页 + if news_id in news_expose_list: + continue + + # TODO 有些新闻可能获取不到静态的信息，这里应该有什么bug + # bug 原因是，json.loads() redis中的数据会报错，需要对redis中的数据进行处理 + # 可以在物料处理的时候过滤一遍，json无法load的新闻 + try: + news_info_dict = self.get_news_detail(news_id) + except Exception as e: + with open("/home/recsys/news_rec_server/logs/news_bad_cases.log", "a+") as f: + f.write(news_id + "\n") + print("there are not news detail info for {}".format(news_id)) + continue + # news_info_str = news_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + # news_info_dict = json.loads(news_info_str) + # 需要确认一下前端接收的json，key需要是单引号还是双引号 + news_info_list.append(news_info_dict) + news_expose_list.add(news_id) + # 注意，原数的key中是包含了类别信息的 + selected_news.append(candiate) + cou += 1 + if cou == 10: + break + + if len(selected_news) > 0: + # 手动删除读取出来的缓存结果, 这个很关键, 返回被删除的元素数量，用来检测是否被真的被删除了 + removed_num = self.reclist_redis_db.zrem(hot_list_user_key, *selected_news) + print("the numbers of be removed:", removed_num) + + # 曝光重新落表 + self._save_user_exposure(user_id,news_expose_list) + #print(news_expose_list, len(news_expose_list)) + # print(news_info_list) + return news_info_list + else: + #TODO 临时这么做，这么做不太好 + self.reclist_redis_db.zunionstore(hot_list_user_key, ["hot_list"]) + print("copy a hot_list for {}".format(hot_list_user_key)) + # 如果是把所有内容都刷完了再重新拷贝的数据，还得记得把今天的曝光数据给清除了 + self.exposure_redis_db.delete(user_exposure_key) + return self.get_hot_list(user_id) + + + def get_news_detail(self, news_id): + """获取新闻展示的详细信息 + """ + # print(1111) + news_info_str = self.static_news_info_redis_db.get("static_news_detail:" + news_id) + # print(222) + # print(news_info_str) + news_info_str = news_info_str.replace('\'', '\"' ) # 将单引号都替换成双引号 + # print(333) + # print(news_info_str) + news_info_dit = json.loads(news_info_str) + # print(444) + + # print("news_info_dit:", news_info_dit) + news_dynamic_info_str = self.dynamic_news_info_redis_db.get("dynamic_news_detail:" + news_id) + news_dynamic_info_str = news_dynamic_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + news_dynamic_info_dit = json.loads(news_dynamic_info_str) + + # print("news_info_dit:", news_dynamic_info_dit) + + for k in news_dynamic_info_dit.keys(): + news_info_dit[k] = news_dynamic_info_dit[k] + + return news_info_dit + + def update_news_dynamic_info(self, news_id,action_type): + """更新新闻展示的详细信息 + """ + news_dynamic_info_str = self.dynamic_news_info_redis_db.get("dynamic_news_detail:" + news_id) + news_dynamic_info_str = news_dynamic_info_str.replace("'", '"' ) # 将单引号都替换成双引号 + news_dynamic_info_dict = json.loads(news_dynamic_info_str) + # print("update",news_id,action_type) + if len(action_type) == 2: + if action_type[1] == "true": + news_dynamic_info_dict[action_type[0]] +=1 + elif action_type[1] == "false": + news_dynamic_info_dict[action_type[0]] -=1 + else: + news_dynamic_info_dict["read_num"] +=1 + # print("update",news_dynamic_info_dict) + news_dynamic_info_str = json.dumps(news_dynamic_info_dict) + # print("update",news_dynamic_info_str) + news_dynamic_info_str = news_dynamic_info_str.replace('"', "'" ) + res = self.dynamic_news_info_redis_db.set("dynamic_news_detail:" + news_id, news_dynamic_info_str) + return res + + def _save_user_exposure(self, user_id, newslist): + """记录用户曝光到redis""" + if len(newslist) == 0: return False # 无曝光数目 + + ctime = str(round(time.time()*1000)) # 曝光时间戳 + + key = "user_exposure:" + str(user_id) # 为key拼接 + + # 将历史曝光记录与newlist(最新曝光)的交集新闻提出来 并将该部分删除，防止重复存储曝光新闻 + exposure_news_set = self.exposure_redis_db.smembers(key) # 历史曝光记录 + + del_exposure_news = [] # 历史曝光记录与newlist(最新曝光)的交集新闻,需要删除 + if exposure_news_set.__len__() != 0: + del_exposure_news = [item for item in exposure_news_set if item.split(":")[0] in newslist] + + # 为曝光过的新闻拼接时间 + news_save = [] + for news_id in newslist: + val = news_id+":"+ctime + val = val.replace('"', "'" ) # 将双引号都替换成单引号 + news_save.append(val) + + # 存储redis + try: + if del_exposure_news.__len__() != 0: + self.exposure_redis_db.srem(key,*del_exposure_news) + self.exposure_redis_db.sadd(key,*news_save) + except Exception as e: + print(str(e)) + return False + return True + + + # def save_user_consume(self, user_id, new_id): + # """ + # 记录用户消费过的新闻 + # """ + # key = "user_consume:" + str(user_id) + # new_id = new_id.replace('"', "'" ) # 将双引号都替换成单引号 + # try: + # self.consume_redis_db.sadd(key,new_id) + # except Exception as e: + # print(str(e)) + # return False + # return True + + +if __name__ == "__main__": + # 测试单例模式 + oneline_server = OnlineServer() + + oneline_server.get_hot_list("4563333734895456257") + + # print(oneline_server.get_hot_list("4563333734895456257")) + # print("***********************") + # print(oneline_server.get_hot_list("4563333734895456257")) diff --git a/codes/news_rec_server/recprocess/recall/__pycache__/hot_recall.cpython-38.pyc b/codes/news_rec_server/recprocess/recall/__pycache__/hot_recall.cpython-38.pyc new file mode 100644 index 00000000..c7a175e6 Binary files /dev/null and b/codes/news_rec_server/recprocess/recall/__pycache__/hot_recall.cpython-38.pyc differ diff --git a/codes/news_rec_server/recprocess/recall/hot_recall.py b/codes/news_rec_server/recprocess/recall/hot_recall.py new file mode 100644 index 00000000..d4a2914c --- /dev/null +++ b/codes/news_rec_server/recprocess/recall/hot_recall.py @@ -0,0 +1,62 @@ +import sys +sys.path.append('../../') + +from dao.mongo_server import MongoServer +from dao.redis_server import RedisServer +from datetime import datetime + + +# 这里需要从物料库中获取物料的信息，然后更新物料当天最新的热度信息 +# 最终将计算好的物料热度，对物料进行排序 + +class HotRecall(object): + def __init__(self): + self.feature_protrail_collection = MongoServer().get_feature_protrail_collection() + self.reclist_redis = RedisServer().get_reclist_redis() + + def get_hot_rec_list(self): + """获取物料的点赞，收藏和创建时间等信息，计算热度并生成热度推荐列表存入redis + """ + # 遍历物料池里面的所有文章 + for item in self.feature_protrail_collection.find(): + news_id = item['news_id'] + news_cate = item['cate'] + news_ctime = item['ctime'] + news_likes_num = item['likes'] + news_collections_num = item['collections'] + news_read_num = item['read_num'] + news_hot_value = item['hot_value'] + + #print(news_id, news_cate, news_ctime, news_likes_num, news_collections_num, news_read_num, news_hot_value) + + # 时间转换与计算时间差 前提要保证当前时间大于新闻创建时间，目前没有捕捉异常 + news_ctime_standard = datetime.strptime(news_ctime, "%Y-%m-%d %H:%M") + cur_time_standard = datetime.now() + time_day_diff = (cur_time_standard - news_ctime_standard).days + time_hour_diff = (cur_time_standard - news_ctime_standard).seconds / 3600 + + # 只要最近3天的内容 + if time_day_diff > 3: + continue + + # 计算热度分，这里使用魔方秀热度公式， 可以进行调整, read_num 上一次的 hot_value 上一次的hot_value用加？ 因为like_num这些也都是累加上来的， 所以这里计算的并不是增值，而是实时热度吧 + # news_hot_value = (news_likes_num * 6 + news_collections_num * 3 + news_read_num * 1) * 10 / (time_hour_diff+1)**1.2 + # 72 表示的是3天， + news_hot_value = (news_likes_num * 0.6 + news_collections_num * 0.3 + news_read_num * 0.1) * 10 / (1 + time_hour_diff / 72) + + #print(news_likes_num, news_collections_num, time_hour_diff) + + # 更新物料池的文章hot_value + item['hot_value'] = news_hot_value + self.feature_protrail_collection.update({'news_id':news_id}, item) + + #print("news_hot_value: ", news_hot_value) + + # 保存到redis中 + self.reclist_redis.zadd('hot_list', {'{}_{}'.format(news_cate, news_id): news_hot_value}, nx=True) + + + + + + diff --git a/codes/news_rec_server/requirements.txt b/codes/news_rec_server/requirements.txt new file mode 100644 index 00000000..cd178d30 --- /dev/null +++ b/codes/news_rec_server/requirements.txt @@ -0,0 +1,57 @@ +async-generator==1.10 +attrs==21.2.0 +Automat==20.2.0 +certifi==2021.10.8 +cffi==1.15.0 +click==8.0.3 +constantly==15.1.0 +cryptography==35.0.0 +cssselect==1.1.0 +EasyProcess==0.3 +Flask==2.0.2 +Flask-Cors==3.0.10 +greenlet==1.1.2 +h11==0.12.0 +h2==3.2.0 +hpack==3.0.0 +hyperframe==5.2.0 +hyperlink==21.0.0 +idna==3.3 +incremental==21.3.0 +itemadapter==0.4.0 +itemloaders==1.0.4 +itsdangerous==2.0.1 +jieba==0.42.1 +Jinja2==3.0.2 +jmespath==0.10.0 +lxml==4.6.3 +MarkupSafe==2.0.1 +outcome==1.1.0 +parsel==1.6.0 +priority==1.3.0 +Protego==0.1.16 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +PyDispatcher==2.0.5 +pymongo==3.12.1 +PyMySQL==1.0.2 +pyOpenSSL==21.0.0 +queuelib==1.6.2 +redis==3.5.3 +Scrapy==2.5.1 +selenium==4.0.0 +service-identity==21.1.0 +six==1.16.0 +sniffio==1.2.0 +sortedcontainers==2.4.0 +SQLAlchemy==1.4.26 +trio==0.19.0 +trio-websocket==0.9.2 +Twisted==21.7.0 +typing-extensions==3.10.0.2 +urllib3==1.26.7 +w3lib==1.22.0 +Werkzeug==2.0.2 +wsproto==1.0.0 +zope.interface==5.4.0 diff --git a/codes/news_rec_server/scheduler/crawl_news.sh b/codes/news_rec_server/scheduler/crawl_news.sh new file mode 100755 index 00000000..0da80e1f --- /dev/null +++ b/codes/news_rec_server/scheduler/crawl_news.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 这个脚本每天凌晨2点30会自动跑 +# 设置python环境 +python=/home/recsys/miniconda3/envs/news_rec_py3/bin/python +news_recsys_path="/home/recsys/news_rec_server" + +# 得跳转到这个目录才能执行下面爬虫的命令 +cd ${news_recsys_path}/materials/news_scrapy + +# 系统正式运行的时候需要修改pages的值 +pages=30 +min_news_num=1000 + +echo "$(date -d today +%Y-%m-%d-%H-%M-%S)" +# 爬虫 +${python} ${news_recsys_path}/materials/news_scrapy/sinanews/run.py --pages=${pages} +if [ $? -eq 0 ]; then + echo "scrapy crawl sina_spider --pages ${page} success." +else + echo "scrapy crawl sina_spider --pages ${page} fail." +fi + + +# 检查今天爬取的数据是否少于min_news_num篇文章，这里也可以配置邮件报警 +${python} ${news_recsys_path}/materials/news_scrapy/monitor_news.py ${min_news_num} +if [ $? -eq 0 ]; then + echo "run python monitor_news.py success." +else + echo "run python monitor_news.py fail." +fi diff --git a/codes/news_rec_server/scheduler/offline_material_and_user_process.sh b/codes/news_rec_server/scheduler/offline_material_and_user_process.sh new file mode 100755 index 00000000..d1cda05c --- /dev/null +++ b/codes/news_rec_server/scheduler/offline_material_and_user_process.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +python=/home/recsys/miniconda3/envs/news_rec_py3/bin/python +news_recsys_path="/home/recsys/news_rec_server" + +echo "$(date -d today +%Y-%m-%d-%H-%M-%S)" + +# 为了更方便的处理路径的问题，可以直接cd到我们想要运行的目录下面 +cd ${news_recsys_path}/materials + +# 更新物料画像 +${python} process_material.py +if [ $? -eq 0 ]; then + echo "process_material success." +else + echo "process_material fail." +fi + +# 更新用户画像 +${python} process_user.py +if [ $? -eq 0 ]; then + echo "process_user.py success." +else + echo "process_user.py fail." +fi + +# 清除前一天redis中的数据，更新最新今天最新的数据 +${python} update_redis.py +if [ $? -eq 0 ]; then + echo "update_redis success." +else + echo "update_redis fail." +fi + + +echo " " + diff --git a/codes/news_rec_server/scheduler/run_offline.sh b/codes/news_rec_server/scheduler/run_offline.sh new file mode 100755 index 00000000..0793542e --- /dev/null +++ b/codes/news_rec_server/scheduler/run_offline.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +python=/home/recsys/miniconda3/envs/news_rec_py3/bin/python +news_recsys_path="/home/recsys/news_rec_server" + +cd ${news_recsys_path}/recprocess + +echo "$(date -d today +%Y-%m-%d-%H-%M-%S)" + +# 离线将推荐列表和热门列表存入redis +${python} offline.py +if [ $? -eq 0 ]; then + echo "run ${python} ${news_recsys_path}/recprocess/offline.py success." +else + echo "run ${python} ${news_recsys_path}/recprocess/offline.py fail." +fi + +echo " " diff --git a/codes/news_rec_server/server.py b/codes/news_rec_server/server.py new file mode 100644 index 00000000..9a1e5524 --- /dev/null +++ b/codes/news_rec_server/server.py @@ -0,0 +1,228 @@ +import sys +sys.path.append("./") +import json +from flask_cors import * +from flask import Flask, jsonify, request +import snowflake.client +from dao.mysql_server import MysqlServer +from dao.entity.register_user import RegisterUser +from dao.entity.logitem import LogItem +from dao.entity.user_likes import UserLikes +from dao.entity.user_collections import UserCollections +from controller.user_action_controller import UserAction +from controller.log_controller import LogController +from recprocess.online import OnlineServer as RecsysServer + +app = Flask(__name__) + +# 允许跨域访问 +CORS(app, supports_credentials=True) + +# 定义推荐服务的实例, 是一个单例类 +recsys_server = RecsysServer() + +@app.route('/recsys/register', methods=["POST"]) +def register(): + """用户注册 + """ + request_str = request.get_data() + request_dict = json.loads(request_str) + # print(request_dict) + + user = RegisterUser() + user.username = request_dict["username"] + user.passwd = request_dict["passwd"] + + # 查询当前用户名是否已经被用过了 + result = UserAction().user_is_exist(user, "register") + + if result != 0: + return jsonify({"code": 500, "mgs": "this username is exists"}) + + user.userid = snowflake.client.get_guid() # 雪花算法 + + user.age = request_dict["age"] + user.gender = request_dict["gender"] + user.city = request_dict["city"] + + # 添加注册用户 + save_res = UserAction().save_user(user) + if not save_res: + return jsonify({"code": 500, "mgs": "register fail."}) + + return jsonify({"code": 200, "msg": "register success."}) + + +@app.route('/recsys/login', methods=["POST"]) +def login(): + """用户登录 + """ + request_str = request.get_data() + request_dict = json.loads(request_str) + + user = RegisterUser() + user.username = request_dict["username"] + user.passwd = request_dict["passwd"] + + # 查询数据库中的用户名或者密码是否存在 + try: + result = UserAction().user_is_exist(user, "login") + # print(result,"login") + if result == 1: + return jsonify({"code": 200, "msg": "login success"}) + elif result == 2: + # 密码错误 + return jsonify({"code": 500, "msg": "passwd is error"}) + else: + return jsonify({"code": 500, "msg": "this username is not exist!"}) + except Exception as e: + return jsonify({"code": 500, "mgs": "login fail."}) + + +@app.route('/recsys/rec_list', methods=["GET"]) +def rec_list(): + """推荐页 + """ + user_name = request.args.get('user_id') + page_id = request.args.get('page_id') + + # 查询用户的id + user_id = UserAction().get_user_id_by_name(user_name) + if not user_id: + return False + + if user_id is None or page_id is None: + return jsonify({"code": 2000, "msg": "user_id or page_id is none!"}) + try: + rec_news_list = recsys_server.get_rec_list(user_id, page_id) + if len(rec_news_list) == 0: + return jsonify({"code": 500, "msg": "rec_list data is empty."}) + return jsonify({"code": 200, "msg": "request rec_list success.", "data": rec_news_list, "user_id": user_id}) + except Exception as e: + print(str(e)) + return jsonify({"code": 500, "msg": "redis fail."}) + + +@app.route('/recsys/hot_list', methods=["GET"]) +def hot_list(): + """热门页面 + """ + if request.method == "GET": + user_name = request.args.get('user_id') + page_id = request.args.get('page_id') + + if user_name is None or page_id is None: + return jsonify({"code": 2000, "msg": "user_name or page_id is none!"}) + + # 查询用户的id + user_id = UserAction().get_user_id_by_name(user_name) + if not user_id: + return False + + try: + # 这里需要改成get_hot_list, 当前get_hot_list方法还没有实现 + rec_news_list = recsys_server.get_hot_list(user_id) + # 下面这个接口是用来前端测试的 + # rec_news_list = recsys_server.get_rec_list(user_id, page_id) + + if len(rec_news_list) == 0: + return jsonify({"code": 200, "msg": "request redis data fail."}) + # rec_news_list = recsys_server.get_hot_list(user_id, page_id) + return jsonify({"code": 200, "msg": "request hot_list success.", "data": rec_news_list, "user_id": user_id}) + except Exception as e: + print(str(e)) + return jsonify({"code": 2000, "msg": "request hot_list fail."}) + + +@app.route('/recsys/news_detail', methods=["GET"]) +def news_detail(): + """一篇文章的详细信息 + """ + user_name = request.args.get('user_name') + news_id = request.args.get('news_id') + + user_id = UserAction().get_user_id_by_name(user_name) + + # if news_id is None or user_id is None: + if news_id is None or user_name is None: + return jsonify({"code": 2000, "msg": "news_id is none or user_name is none!"}) + try: + news_detail = recsys_server.get_news_detail(news_id) + + # recsys_server.save_user_consume(user_id,news_id) # 记录用户消费的 + + if UserAction().get_likes_counts_by_user(user_id,news_id) > 0: + news_detail["likes"] = True + else: + news_detail["likes"] = False + + if UserAction().get_coll_counts_by_user(user_id,news_id) > 0: + news_detail["collections"] = True + else: + news_detail["collections"] = False + # print("test",news_detail) + return jsonify({"code": 0, "msg": "request news_detail success.", "data": news_detail}) + except Exception as e: + print(str(e)) + return jsonify({"code": 2000, "msg": "error"}) + + +@app.route('/recsys/action', methods=["POST"]) +def actions(): + """用户的行为：阅读，点赞，收藏 + """ + request_str = request.get_data() + request_dict = json.loads(request_str) + + username = request_dict.get('user_name') + newsid = request_dict.get('news_id') + actiontype = request_dict.get("action_type") + actiontime = request_dict.get("action_time") + + userid = UserAction().get_user_id_by_name(username) # 获取用户 id + if not userid: + return jsonify({"code": 2000, "msg": "user not register"}) + + #TODO 先判断当前的action_type是否是取消的意思，如果是的话，需要将数据库中对应的操作删掉 + action_type_list = actiontype.split(":") + # print(actiontype) + if len(action_type_list) == 2: + _action_type = action_type_list[0] + if action_type_list[1] == "false": # 如果数据库中这个参数为false的话 + # 删除数据 + if _action_type=="likes": + UserAction().del_likes_by_user(userid,newsid) # 删除用户喜欢记录 + elif _action_type=="collections": + UserAction().del_coll_by_user(userid,newsid) # 删除用户收藏记录 + else: + if _action_type=="likes": + userlikes = UserLikes() + userlikes.new(userid,username,newsid) + UserAction().save_one_action(userlikes) # 记录用户喜欢记录 + elif _action_type=="collections": + usercollections = UserCollections() + usercollections.new(userid,username,newsid) + UserAction().save_one_action(usercollections) # 记录用户收藏记录 + + try: + # 落日志 + logitem = LogItem() + logitem.new(userid,newsid,action_type_list[0]) + LogController().save_one_log(logitem) + + # 更新redis中的展示数据 新闻侧 + # if action_type_list[0] in ["read","likes","collections"]: + recsys_server.update_news_dynamic_info(news_id=newsid,action_type=action_type_list) + return jsonify({"code": 200, "msg": "action success"}) + + except Exception as e: + print(str(e)) + return jsonify({"code": 2000, "msg": "action error"}) + + +if __name__ == '__main__': + # 允许服务器被公开访问 + app.run(debug=True, host='0.0.0.0', port=3000, threaded=True) + # 只能被自己的机子访问 + # app.run(debug=True, host='127.0.0.1', port=10086, threaded=True) + diff --git a/codes/news_rec_web/Vue-newsinfo b/codes/news_rec_web/Vue-newsinfo new file mode 160000 index 00000000..e8ade14f --- /dev/null +++ b/codes/news_rec_web/Vue-newsinfo @@ -0,0 +1 @@ +Subproject commit e8ade14ff0e4c7d536ddc8baf201a948a6c25a18 diff --git a/codes/news_rec_web/node-v8.17.0-linux-x64.tar b/codes/news_rec_web/node-v8.17.0-linux-x64.tar new file mode 100644 index 00000000..cc8bd8b1 Binary files /dev/null and b/codes/news_rec_web/node-v8.17.0-linux-x64.tar differ