持久化聊天数据到redis

This commit is contained in:
linghaihui 2023-04-04 22:21:42 +08:00
parent 1b8d864b39
commit 68fd5a79d7
12 changed files with 275 additions and 64 deletions

View File

@ -1,5 +1,6 @@
import {
doRequest
doRequest,
sid_prefix
} from "./config"
App({
@ -8,8 +9,45 @@ App({
this.getSid(sid => {
console.log(sid)
})
this.upload_conversation()
},
onHide: function () {
this.upload_conversation()
},
globalData: {},
upload_cache_conversation: function (sid) {
wx.getStorage({
key: "chatList",
success: function (res) {
var data = res.data
if (data && data.length > 0) {
doRequest("/save", "POST", {
"sid": sid_prefix + sid,
"conversations": data
}).then(res => {
console.log("upload " + data.length + " conversations success!")
})
}
}
})
},
upload_conversation: function (conversations = []) {
var that = this
if (conversations.length == 0) {
that.getSid(sid => {
that.upload_cache_conversation(sid)
})
} else {
that.getSid(sid => {
doRequest("/save", "POST", {
"sid": sid_prefix + sid,
"conversations": conversations,
}).then(res => {
console.log("upload " + conversations.length + " conversations success!")
})
})
}
},
getSid: function (callback) {
var that = this
if (!this.globalData.sid) {

View File

@ -1,5 +1,11 @@
const app = getApp()
import {
doRequest,
sid_prefix,
systemInfo
} from "../../config"
var closeShareOnCopy = false
try {
if (wx.getStorageSync("closeShareOnCopy")) {
@ -22,14 +28,14 @@ Component({
},
pageLifetimes: {
show: function () {
this.initMessageHistory()
// this.initMessageHistory()
},
},
lifetimes: {
attached() {
var that = this
app.globalData.cht = that
//that.initMessageHistory()
that.initMessageHistory()
wx.getSystemInfo({
success: function (res) {
that.setData({
@ -48,22 +54,69 @@ Component({
autoIncrConversation: 1,
closeShareOnCopy: closeShareOnCopy,
showShare: false,
loadingData: false,
height: systemInfo.windowHeight - parseInt(100 / 750 * systemInfo.windowWidth) - ((systemInfo.platform == "ios" || systemInfo.platform == "android") ? 22 : 5)
},
methods: {
initMessageHistory() {
bindscrolltoupper: function (e) {
var that = this
var data = wx.getStorageSync("chatList")
data = data ? data : []
data.forEach((v) => {
if (v["suggests"] === undefined) {
v["suggests"] = []
if (that.data.loadingData) {
return
}
})
if (data.length > 0) {
that.setData({
chatList: data,
loadingData: true
})
wx.showLoading({
title: "加载历史记录...",
})
app.getSid(sid => {
var page = 1
if (that.data.chatList.length > 0) {
page = Math.ceil((that.data.chatList.length + 1) / 10)
}
doRequest("/query", "GET", {
"sid": sid_prefix + sid,
"page": page,
"size": 10,
}).then(res => {
var data = res.data["data"]
data.reverse()
var oldData = that.data.chatList
var filterData = []
if (oldData.length > 0) {
data.forEach(k => {
if (k["dt"] < oldData[0]["dt"]) {
filterData.push(k)
}
})
} else {
filterData = data
}
var newData = filterData.concat(oldData)
that.setData({
chatList: newData,
loadingData: false
}, () => {
if (filterData.length == 0 && e) {
setTimeout(() => {
wx.showToast({
title: "已加载完成"
})
}, 300)
} else {
setTimeout(() => {
wx.hideLoading()
}, 300)
}
})
}).catch(res => {
wx.hideLoading()
console.log(res)
})
})
},
initMessageHistory() {
this.bindscrolltoupper()
},
clearChat: function (e) {
var that = this
@ -73,13 +126,18 @@ Component({
content: "是否删除该条聊天?",
complete: (res) => {
if (res.confirm) {
var deleteData = data[index]
data.splice(index, 1)
that.setData({
chatList: data,
})
wx.setStorage({
key: "chatList",
data: data,
app.getSid(sid => {
doRequest("/delete", "POST", {
"sid": sid_prefix + sid,
"conversation": deleteData
}).then(res => {
console.log(res)
})
})
}
},

View File

@ -1,11 +1,11 @@
<wxs src="../../tools.wxs" module="tools" />
<view wx:if="{{chatList.length == 0}}" style="text-align:center;color: #b4bbc4;font-size: 30rpx;">输入问题开始和{{chatType == "bing" ? "New Bing" : "ChatGPT"}}聊天吧~</view>
<scroll-view class="chat" scroll-y="{{true}}" scroll-into-view="{{scrollId}}" style="height:{{systemInfo.windowHeight - 70}}px;" enable-back-to-top="{{true}}" scroll-anchoring="{{true}}" enhanced="{{true}}" enable-passive="{{true}}" show-scrollbar="{{false}}" enable-flex="{{true}}">
<scroll-view class="chat" scroll-y="{{true}}" scroll-into-view="{{scrollId}}" style="height:{{height}}px;" enable-back-to-top="{{true}}" scroll-anchoring="{{true}}" enhanced="{{true}}" show-scrollbar="{{false}}" enable-flex="{{true}}" bindrefresherrefresh="bindscrolltoupper" scroll-with-animation="{{true}}" refresher-enabled="{{ systemInfo.platform == 'ios' || systemInfo.platform == 'android'}}" refresher-triggered="{{loadingData}}" refresher-threshold="80" bindscrolltoupper="{{!(systemInfo.platform == 'ios' || systemInfo.platform == 'android') ? 'bindscrolltoupper': ''}}">
<view wx:for="{{chatList}}" wx:key="index" wx:for-item="item" id="{{'item'+index}}">
<view class="chat-item left" wx:if="{{item.type != 'man'}}" id="msg-{{index}}">
<image class="avatar" src="{{item.avatarUrl}}" style="display: flex;" catchlongpress="clearChat" data-index="{{index}}" catchtap="showOriginContent" data-index="{{index}}"></image>
<view class="chat-box" style="margin-left: 20rpx;">
<view style="display: flex;flex-direction: row;align-items: center;"><text class="dt" style="flex: 1;">{{item.dt}}</text>
<view style="display: flex;flex-direction: row;align-items: center;"><text class="dt" style="flex: 2;">{{item.dt}}</text>
<view wx:if="{{item.num_in_conversation && item.num_in_conversation != -1}}" style="display:flex;justify-content:flex-end; flex: 1;align-items: center;"><text class="conversation_num">{{item.num_in_conversation}}</text></view>
</view>
<view class="content bg-white" catchlongpress="copyContent" data-index="{{index}}" catchtap="{{tools.indexOf(item.originContent, '```markdown') ? 'renderMd': ''}}">
@ -28,5 +28,5 @@
</view>
<view id="{{'item'+ autoIncrConversation + 9999}}" style="height: 1em;"></view>
</scroll-view>
<icon wx:if="{{receiveData}}" type="cancel" catchtap="cancelReceive" style="position: absolute;bottom: 145rpx;right:1%;z-index: 10000;" size="20"></icon>
<icon wx:if="{{receiveData}}" type="cancel" catchtap="cancelReceive" style="position: absolute;bottom: 148rpx;right:1%;z-index: 10000;" size="22"></icon>
<popup message="是否分享搜索内容?" wx:if="{{showShare}}" bindPopButtonClick="onPopButtonClick" openType="share"></popup>

View File

@ -1,11 +1,10 @@
import {
doRequest,
SERVER_WSS_HOST
SERVER_WSS_HOST,
systemInfo,
sid_prefix
} from "../../config"
const systemInfo = wx.getSystemInfoSync()
// 各平台对话分离
const sid_prefix = systemInfo.platform == "ios" || systemInfo.platform == "android" ? "" : systemInfo.platform
const initHeight = inputPop() ? 22 : 5
// 是否使用websocket请求
var useWebsocket = true
@ -146,12 +145,6 @@ Page({
chatType: chatType,
})
}
const cht = app.globalData.cht
if (cht.data.chatList.length > 1) {
cht.setData({
scrollId: "item" + (cht.data.chatList.length - 2),
})
}
// 切换title
this.switchTitle()
},
@ -166,7 +159,24 @@ Page({
})
}
},
onLoad() {},
scrollBottom: function () {
const cht = app.globalData.cht
if (cht.data.chatList.length > 1 && !this.data.textareaFocus) {
cht.setData({
scrollId: "item" + (cht.data.chatList.length - 2),
})
}
},
onLoad() {
const cht = app.globalData.cht
setTimeout(() => {
if (cht.data.chatList.length > 1) {
cht.setData({
scrollId: "item" + (cht.data.chatList.length - 2),
})
}
}, 1500)
},
processData: function (data, suggests, content) {
var robContent = data["data"]["status"]
if (robContent == "Success") {
@ -268,7 +278,7 @@ Page({
})
return
} else {
that.pushStorageMessage(cht, "搜索中🔍...", "rob", [], true)
that.pushStorageMessage(cht, "搜索中🔍...", "rob", [], true, false, -1, false)
}
if (that.data.useWebsocket) {
that.sendWSRequest(content)
@ -300,10 +310,14 @@ Page({
searching: false
})
}
// 只保留最新的10条
wx.setStorage({
key: "chatList",
data: cht.data.chatList,
data: cht.data.chatList.slice(cht.data.chatList.length - 10),
})
if (final) {
app.upload_conversation(cht.data.chatList.slice(cht.data.chatList.length - 1))
}
setTimeout(() => {
cht.setData({
scrollId: "item" + (autoIncrConversation + "9999"),
@ -537,9 +551,12 @@ Page({
cht.setData({
chatList: [],
})
wx.setStorage({
key: "chatList",
data: [],
app.getSid(sid => {
doRequest("/delete_all", "POST", {
"sid": sid_prefix + sid
}).then(res => {
console.log("delete all")
})
})
}
},

View File

@ -1,6 +1,6 @@
<chat-box bindsuggestSubmit="onSuggestSubmit" bindcancelReceive="onCancelReceive" bindswitchRequestMethod="switchRequestMethod" catchlongpress="longPress" chatType="{{chatType}}"></chat-box>
<view style="bottom:{{inputBottom}}px; border-radius: 20rpx;margin-left: 1%;width: 98%;min-height: 100rpx;position: fixed;background-color: #f4f6f8;display: flex;align-items:flex-start; justify-content: space-between;{{textareaFocus ? 'border: 1px solid #b4bbc4;': ''}}">
<textarea bindfocus="inputFocus" bindblur="inputBlur" value="{{content}}" adjust-position="{{false}}" focus="{{textareaFocus}}" maxlength="2000" auto-height="{{true}}" cursor-spacing="10" bindconfirm="submit" fixed="{{true}}" show-confirm-bar="{{false}}" confirm-type="send" placeholder="{{systemInfo.platform == 'mac' || systemInfo.platform == 'windows' ? '请输入问题,输入>>>提交...': '请输入问题...'}}" style="padding: 10rpx;flex: 9;line-height: normal;" placeholder-style="color: #b4bbc4" catchtap="focus" bindinput="inputData"></textarea>
<textarea bindfocus="inputFocus" bindblur="inputBlur" value="{{content}}" adjust-position="{{false}}" focus="{{textareaFocus}}" maxlength="2000" auto-height="{{true}}" cursor-spacing="10" bindconfirm="submit" fixed="{{true}}" show-confirm-bar="{{false}}" confirm-type="send" placeholder="{{systemInfo.platform == 'mac' || systemInfo.platform == 'windows' ? '请输入问题,输入>>>提交...': '请输入问题...'}}" style="padding: 10rpx;flex: 9;line-height: normal;" placeholder-style="color: #b4bbc4" catchtap="focus" bindinput="inputData" catchlongpress="scrollBottom"></textarea>
<view style="background-color: #f4f6f8;color: {{content ? black : '#b4bbc4'}};border-radius: 0 20rpx 20rpx 0;height: 90rpx;cursor: pointer;margin-right: 15rpx;padding-top:10rpx;font-size: 32rpx;" catchtap="submit" wx:if="{{systemInfo.platform != 'ios'}}">发送</view>
</view>
<popup message="{{searchPopMessage}}" wx:if="{{showSearchPop}}" bindPopButtonClick="onPopButtonClick" data-q="{{q}}"></popup>

View File

@ -2,7 +2,7 @@ FROM sanicframework/sanic:3.11-latest
WORKDIR /sanic
COPY app.py EdgeGPT.py requirements.txt /sanic/
COPY app.py EdgeGPT.py conversation_ctr.py requirements.txt /sanic/
RUN pip install -r requirements.txt

View File

@ -22,20 +22,20 @@ HEADERS = {
"accept": "application/json",
"accept-language": "en-US,en;q=0.9",
"content-type": "application/json",
"sec-ch-ua": '"Not_A Brand";v="99", "Microsoft Edge";v="110", "Chromium";v="110"',
"sec-ch-ua": '"Microsoft Edge";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
"sec-ch-ua-arch": '"x86"',
"sec-ch-ua-bitness": '"64"',
"sec-ch-ua-full-version": '"109.0.1518.78"',
"sec-ch-ua-full-version-list": '"Chromium";v="110.0.5481.192", "Not A(Brand";v="24.0.0.0", "Microsoft Edge";v="110.0.1587.69"',
"sec-ch-ua-full-version": '"111.0.1661.43"',
"sec-ch-ua-full-version-list": '"Microsoft Edge";v="111.0.1661.43", "Not(A:Brand";v="8.0.0.0", "Chromium";v="111.0.5563.64"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-model": "",
"sec-ch-ua-platform": '"Windows"',
"sec-ch-ua-platform-version": '"15.0.0"',
"sec-ch-ua-platform": '"macOS"',
"sec-ch-ua-platform-version": '"11.7.3"',
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"x-ms-client-request-id": str(uuid.uuid4()),
"x-ms-useragent": "azsdk-js-api-client-factory/1.0.0-beta.1 core-rest-pipeline/1.10.0 OS/Win32",
"x-ms-useragent": "azsdk-js-api-client-factory/1.0.0-beta.1 core-rest-pipeline/1.10.0 OS/MacIntel",
"Referer": "https://www.bing.com/search?q=Bing+AI&showconv=1&FORM=hpcodx",
"Referrer-Policy": "origin-when-cross-origin",
"x-forwarded-for": FORWARDED_IP,
@ -46,21 +46,21 @@ HEADERS_INIT_CONVER = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "en-US,en;q=0.9",
"cache-control": "max-age=0",
"sec-ch-ua": '"Chromium";v="110", "Not A(Brand";v="24", "Microsoft Edge";v="110"',
"sec-ch-ua": '"Microsoft Edge";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
"sec-ch-ua-arch": '"x86"',
"sec-ch-ua-bitness": '"64"',
"sec-ch-ua-full-version": '"110.0.1587.69"',
"sec-ch-ua-full-version-list": '"Chromium";v="110.0.5481.192", "Not A(Brand";v="24.0.0.0", "Microsoft Edge";v="110.0.1587.69"',
"sec-ch-ua-full-version": '"111.0.1661.43"',
"sec-ch-ua-full-version-list": '"Microsoft Edge";v="111.0.1661.43", "Not(A:Brand";v="8.0.0.0", "Chromium";v="111.0.5563.64"',
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-model": '""',
"sec-ch-ua-platform": '"Windows"',
"sec-ch-ua-platform-version": '"15.0.0"',
"sec-ch-ua-platform": '"macOS"',
"sec-ch-ua-platform-version": '"11.7.3"',
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
"sec-fetch-user": "?1",
"upgrade-insecure-requests": "1",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.69",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.43",
"x-edge-shopping-flag": "1",
"x-forwarded-for": "1.1.1.1",
}
@ -288,8 +288,6 @@ class ChatHub:
elif response.get("type") == 2:
final = True
yield True, response
else:
print(response)
async def __initial_handshake(self):
await self.wss.send(append_identifier({
@ -367,4 +365,5 @@ class Chatbot:
Reset the conversation
"""
await self.close()
HEADERS["x-ms-client-request-id"] = str(uuid.uuid4())
self.chat_hub = ChatHub(Conversation())

View File

@ -13,6 +13,7 @@ from sanic import Sanic
from sanic.log import logger
from sanic.response import json
from conversation_ctr import conversation_ctr
from EdgeGPT import Chatbot, ConversationStyle
APPID = os.environ.get('WXAPPID')
@ -80,9 +81,10 @@ async def ws_chat(_, ws):
while True:
try:
data = raw_json.loads(await ws.recv())
logger.info('Websocket receive data: %s', data)
logger.info('[bing] Websocket receive data: %s', data)
sid = data['sid']
q = data['q']
index = 0
async for response in get_bot(sid).ask_stream(q, conversation_style=ConversationStyle.creative):
final, res = response
if final:
@ -96,6 +98,8 @@ async def ws_chat(_, ws):
'data': processed_data
}))
else:
index += 1
if index % 3 == 1:
await ws.send(raw_json.dumps({
'final': final,
'data': res
@ -127,7 +131,7 @@ async def reset_conversation(sid):
async def do_chat(request):
logger.info('Http request payload: %s', request.json)
logger.info('[bing] Http request payload: %s', request.json)
return await get_bot(request.json.get('sid')).ask(
request.json.get('q'), conversation_style=ConversationStyle.creative
)
@ -157,9 +161,6 @@ async def process_data(res, q, sid, auto_reset=None):
text = '抱歉,未搜索到结果。'
logger.error('响应异常:%s', res)
suggests = [q]
if res['type'] == 2:
await reset_conversation(sid)
text += '\n已结束本轮对话。'
msg = res['item']['result']['message'] if 'message' in res['item']['result'] else ''
if auto_reset and ('New topic' in text or 'has expired' in msg):
await reset_conversation(sid)
@ -204,7 +205,7 @@ async def ws_openai_chat(_, ws):
while True:
try:
data = raw_json.loads(await ws.recv())
logger.info('Websocket receive data: %s', data)
logger.info('[openai] Websocket receive data: %s', data)
sid = data['sid']
q = data['q']
# 保存30个对话
@ -220,11 +221,14 @@ async def ws_openai_chat(_, ws):
stream=True,
)
chunks = []
index = 0
for chunk in response:
chunk_message = chunk['choices'][0]['delta']
if chunk_message:
if 'content' in chunk_message:
chunks.append(chunk_message['content'])
index += 1
if index % 5 == 1:
await ws.send(
raw_json.dumps({
'final': False,
@ -253,6 +257,7 @@ async def ws_openai_chat(_, ws):
@app.post('/openai_chat')
async def openai_chat(request):
try:
logger.info('[openai] Http request payload: %s', request.json)
sid = request.json.get('sid')
q = request.json.get('q')
history_conversation = OPENAI_CONVERSATION[sid][-30:]
@ -283,5 +288,36 @@ async def openai_chat(request):
return json(make_response_data('Error', str(e), [], str(e)))
@app.route('/last_sync_time')
async def last_sync_time(request):
return json({'last_sync_time': conversation_ctr.get_last_sync_time(request.args.get('sid'))})
@app.post('/save')
async def save(request):
conversation_ctr.save(request.json.get('sid'), request.json.get('conversations'))
return json({})
@app.route('/query')
async def query(request):
data = conversation_ctr.get_by_page(
request.args.get('sid'), int(request.args.get('page', '1')), int(request.args.get('size', '20'))
)
return json({'data': data})
@app.post('/delete')
async def delete(request):
conversation_ctr.delete(request.json.get('sid'), request.json.get('conversation'))
return json({})
@app.post('/delete_all')
async def delete_all(request):
conversation_ctr.delete_all(request.json.get('sid'))
return json({})
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8000)

View File

@ -0,0 +1,58 @@
# coding=utf-8
import json
import os
import redis
REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1')
REDIS_PORT = int(os.environ.get('REDIS_PORT', 6379))
REDIS_PASSWD = os.environ.get('REDIS_PASSWD', '123456')
REDIS_DB = int(os.environ.get('REDIS_DB', 0))
class ConversationCtr:
LAST_SYNC_TIME_KEY = 'bing:last_sync_time:%s'
CONVERSATION_LIST_KEY = 'bing:conversation_list:%s'
def __init__(self, client=None) -> None:
self.redis_client = client
if client is None:
self.init()
def init(self):
if self.redis_client is not None:
return
pool = redis.ConnectionPool(host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWD)
self.redis_client = redis.Redis(connection_pool=pool)
def get_last_sync_time(self, sid):
key = self.LAST_SYNC_TIME_KEY % sid
v = self.redis_client.get(key)
return v.decode() if v else ''
def get_by_page(self, sid, page=1, size=20):
offset = size * (page - 1) if page > 0 else 0
key = self.CONVERSATION_LIST_KEY % sid
return [json.loads(x) for x in self.redis_client.lrange(key, offset, offset + size - 1)]
def save(self, sid, conversations):
_last_sync_time = self.get_last_sync_time(sid)
conversations = [x for x in conversations if x['dt'] > _last_sync_time]
if len(conversations) <= 0:
return
key = self.CONVERSATION_LIST_KEY % sid
self.redis_client.lpush(key, *[json.dumps(x) for x in conversations])
self.redis_client.set(self.LAST_SYNC_TIME_KEY % sid, conversations[-1]['dt'])
def delete(self, sid, conversation):
key = self.CONVERSATION_LIST_KEY % sid
self.redis_client.lrem(key, 0, json.dumps(conversation))
def delete_all(self, sid):
key = self.CONVERSATION_LIST_KEY % sid
self.redis_client.delete(key)
conversation_ctr = ConversationCtr()

View File

@ -6,3 +6,6 @@ COOKIE_FILE2=/sanic/cookies/cookie2.json # 备用cookie2, 没有可以删掉
COOKIE_FILES=["/sanic/cookies/cookie.json", "/sanic/cookies/cookie1.json", "/sanic/cookies/cookie2.json"] #cookie列表配置了此环境变量会优先使用此变量直接忽略上面的3个环境变量
https_proxy=http://127.0.0.1:1080 # 目前中国大陆的IP会返回404所以最好能加个代理
OPENAI_API_KEY=
REDIS_HOST=
REDIS_PORT=
REDIS_PASSWD=

View File

@ -4,3 +4,5 @@ asyncio==3.4.3
websockets==10.4
httpx==0.23.3
openai==0.27.2
redis==4.5.1
hiredis==2.2.2