Compare commits

..

No commits in common. "main" and "producer-dev" have entirely different histories.

15 changed files with 34 additions and 797 deletions

View File

@ -1,17 +0,0 @@
FROM python:3.12
# 设置工作目录
WORKDIR /app
# 将当前目录内容复制到位于容器的/app目录下
COPY . /app
# 安装依赖
RUN pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
RUN pip install --no-cache-dir -r requirements.txt
# 暴露容器的端口
EXPOSE 8000
# 在容器启动时运行app.py
CMD ["python", "main.py"]

View File

@ -1,14 +1,14 @@
[server]
listen = 0.0.0.0
port = 8000
port = 8080
debug = true
[database]
host = 43.140.205.103
host = 172.16.5.2
port = 3306
database = kaku
user = kaku
password = p4J7fY8mc6hcZfjG
database = test
user = root
password = 123456
[kafka]
bootstrap_servers = 172.16.5.2:9092

View File

@ -1,25 +0,0 @@
from dao.db.util import *
def get_orders_count():
# 在mysql中读取统计订单数量
connect = get_connet()
sql = "SELECT * FROM order_name_yn LIMIT 100"
cursor = connect.cursor()
cursor.execute(sql)
result = cursor.fetchall()
cursor.close()
connect.close()
resu = []
for row in result:
resu.append(
{
"order_name": row[0],
"update_time": row[1],
"order_count": row[2]
}
)
return resu

View File

@ -1,70 +1,13 @@
from conf.util import get_config_object
from kafka import TopicPartition
import kafka
import json, re
from kafka import KafkaConsumer
conf = get_config_object()
# Kafka
def get_KafkaConsumer(topic: str) -> kafka.KafkaConsumer:
consumer = kafka.KafkaConsumer(
topic,
bootstrap_servers=conf.get("kafka","bootstrap_servers"), # Kafka 服务器地址
group_id='test', # 消费者组
auto_offset_reset='earliest', # 从最早的消息开始消费
enable_auto_commit=True, # 自动提交消费位移
def get_KafkaConsumer() -> KafkaConsumer:
""" 返回KafkaConsumer对象 """
consumer = KafkaConsumer(
bootstrap_servers=conf.get("kafka", "bootstrap_servers"),
group_id=conf.get("kafka", "group_id")
)
return consumer
def raw_Data_to_jsonstr(data: str) -> str:
"""
将原始数据切分转换为json字符串
"""
# 清理转义字符
data = re.sub(r"\\", "", data)
# 去除多余的空格和换行符
data = data.strip()
data_list = data.split("\t")
return {
"order_id": data_list[0],
"order_category": data_list[1],
"order_name": data_list[2],
"order_quantity": data_list[3],
"date": data_list[4],
"is_valid": data_list[5],
}
def get_offsets(topic_name: str):
"""获取 Kafka 主题的已提交位移和终末位移"""
consumer = get_KafkaConsumer(topic_name)
offsets_data = None
# 获取该主题的所有分区
partitions = consumer.partitions_for_topic(topic_name)
if not partitions:
return print({"error": f"Topic {topic_name} not found"})
# 获取每个分区的已提交位移和终末位移
for partition in partitions:
tp = TopicPartition(topic_name, partition)
# 获取已提交的位移
commit_offset = consumer.committed(tp)
# 获取终末位移high watermark
end_offset = next(iter(consumer.end_offsets([tp]).values()))
offsets_data = {
"partition": partition,
"commit_offset": commit_offset,
"end_offset": end_offset,
"lag": end_offset - commit_offset if commit_offset is not None else None,
}
return offsets_data
return consumer

View File

@ -16,7 +16,7 @@ app = Flask(__name__)
conf = get_config_object()
# 注册路由
app.register_blueprint(api_bp,url_prefix='/api')
app.register_blueprint(api_bp)
app.register_blueprint(page_bp)
# 启动

View File

@ -1,4 +1,3 @@
Flask
pymysql
confluent-kafka
kafka-python-ng
Flask == 3.1.0
pymysql == 1.1.1
confluent-kafka == 2.7.0

View File

@ -1,102 +1,4 @@
from flask import Blueprint, jsonify
from dao.kafka.util import *
from kafka import KafkaConsumer, TopicPartition
from dao.db.mysql import *
from flask import Blueprint
api_bp = Blueprint('api', __name__)
@api_bp.route('/rawdata', methods=['GET'])
def readKafka():
consumer: KafkaConsumer = get_KafkaConsumer("orders")
messages = []
try:
# 读取消息最多读取10条消息
msg = consumer.poll(timeout_ms=1000, max_records=50)
for partition, msgs in msg.items():
for message in msgs:
messages.append(raw_Data_to_jsonstr(message.value.decode('utf-8')))
except Exception as e:
return jsonify({"error": str(e)}), 500
finally:
# 取消订阅并关闭消费者
consumer.close()
return jsonify(messages)
@api_bp.route('/stats/<topic>')
def stats(topic:str):
# 获取Kafka Topic的offset信息
info = get_offsets(topic)
if info is None:
return jsonify({"error": "Topic not found"}), 404
return jsonify(info)
@api_bp.route('/orders-count')
def orders_count():
# 在mysql中读取统计订单数量
return jsonify(get_orders_count())
@api_bp.route('/stream/ordersummary')
def orders_count_by_name():
consumer: KafkaConsumer = get_KafkaConsumer("eachOrders_summary")
messages = []
try:
# 读取消息最多读取10条消息
msg = consumer.poll(timeout_ms=1000, max_records=50)
for partition, msgs in msg.items():
for message in msgs:
jsondata = json.loads(message.value.decode('utf-8'))
messages.append(jsondata)
except Exception as e:
return jsonify({"error": str(e)}), 500
finally:
# 取消订阅并关闭消费者
consumer.close()
print(messages)
return jsonify(messages)
@api_bp.route('/stream/ordernamecount')
def order_name_count():
consumer: KafkaConsumer = get_KafkaConsumer("order_name_count")
messages = []
try:
# 读取消息最多读取10条消息
msg = consumer.poll(timeout_ms=1000, max_records=50)
for partition, msgs in msg.items():
for message in msgs:
jsondata = json.loads(message.value.decode('utf-8'))
messages.append(jsondata)
except Exception as e:
return jsonify({"error": str(e)}), 500
finally:
# 取消订阅并关闭消费者
consumer.close()
return jsonify(messages)
@api_bp.route('/stream/summary')
def summary():
consumer: KafkaConsumer = get_KafkaConsumer("orders_summary")
messages = []
try:
# 读取消息最多读取10条消息
msg = consumer.poll(timeout_ms=1000, max_records=50)
for partition, msgs in msg.items():
for message in msgs:
jsondata = json.loads(message.value.decode('utf-8'))
messages.append(jsondata)
except Exception as e:
return jsonify({"error": str(e)}), 500
finally:
# 取消订阅并关闭消费者
consumer.close()
return jsonify(messages)

View File

@ -6,23 +6,3 @@ page_bp = Blueprint('page', __name__)
@page_bp.route('/')
def index():
return render_template('index.html')
@page_bp.route('/show')
def test():
return render_template('show.html')
@page_bp.route('/ordercount')
def ordercount():
return render_template('ordercount.html')
@page_bp.route('/streamordersummary')
def streamodersummary():
return render_template('streamordersummary.html')
@page_bp.route('/streamordernamecount')
def streamodernamecount():
return render_template('streamordernamecount.html')
@page_bp.route('/streamsummary')
def streamsummary():
return render_template('streamsummary.html')

View File

@ -53,7 +53,7 @@ def delivery_report(err, msg):
if err is not None:
print('Message delivery failed: {}'.format(err))
else:
# print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
pass
def run_kafka_producer():
@ -61,4 +61,4 @@ def run_kafka_producer():
order_data = generate_order_data() # 生成数据
producer.produce('orders', order_data, callback=delivery_report) # 发送到 Kafka 的 orders 主题
producer.poll(0) # 处理任何待处理的事件(如回调)
time.sleep(random.random()*3) # 每隔 1-5 秒发送一次
time.sleep(5) # 每隔 5 秒发送一次

View File

@ -3,81 +3,21 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>订单信息实时统计系统</title>
<!-- 引入Bootstrap的CSS文件 -->
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha1/dist/css/bootstrap.min.css" rel="stylesheet">
<style>
/* 为iframe添加适当的边距和样式 */
iframe {
border-radius: 8px; /* 圆角 */
box-shadow: 0 4px 10px rgba(0, 0, 0, 0.1); /* 阴影 */
width: 100%; /* 自适应宽度 */
height: 400px; /* 固定高度 */
}
/* 给容器添加一些间距 */
.container-fluid {
margin-top: 30px; /* 顶部间距 */
}
/* 设置页面标题的样式 */
h1 {
text-align: center;
margin-bottom: 30px;
}
/* 设置Chart容器的样式 */
.chart-container {
margin-bottom: 30px; /* 每个图表容器之间的间距 */
}
</style>
<title>哇哦哦哦哦哦哦哦哦哦哦哦哦哦哦哦哦</title>
</head>
<body>
<!-- 页面容器 使用container-fluid -->
<div class="container-fluid mt-5">
<!-- 第一个图表容器放在最上面 -->
<div class="row">
<div class="col-12 chart-container">
<iframe src="/show" frameborder="0"></iframe>
</div>
</div>
<!-- 其他四个图表容器,分成两列一行 -->
<div class="row">
<!-- 第二个图表容器 -->
<div class="col-lg-6 col-md-6 col-sm-12 chart-container">
<iframe src="/ordercount" frameborder="0"></iframe>
</div>
<!-- 第三个图表容器 -->
<div class="col-lg-6 col-md-6 col-sm-12 chart-container">
<iframe src="/streamordersummary" frameborder="0"></iframe>
</div>
</div>
<div class="row">
<!-- 第四个图表容器 -->
<div class="col-lg-6 col-md-6 col-sm-12 chart-container">
<iframe src="/streamordernamecount" frameborder="0"></iframe>
</div>
<!-- 第五个图表容器 -->
<div class="col-lg-6 col-md-6 col-sm-12 chart-container">
<iframe src="/streamsummary" frameborder="0"></iframe>
</div>
</div>
<div class="row">
<h2>制作人员名单</h2>
<p>郭子奇架构设计服务器部署和Python代码调试</p>
<p>许家禾spark streaming&spark core/rdd</p>
<p>李尧宇环境部署在Kafka中创建主题order用Python代码实现producer并每隔5秒推送数据给Kafka的order主题</p>
<p>李烁升使用spark streaming每隔两秒实时统计所有订单类别的数量</p>
<p>郭志胜:数据生产及数据展示</p>
<p>陈楠使用spark sql统计各个订单的有效数和无效数量</p>
</div>
</div>
<!-- 引入Bootstrap的JS文件 -->
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.6/dist/umd/popper.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha1/dist/js/bootstrap.min.js"></script>
Lorem ipsum dolor sit amet consectetur adipisicing elit. Culpa magnam dicta harum sit voluptas, explicabo sed cumque omnis. Culpa, reiciendis numquam atque quod id molestiae nobis similique placeat eos amet.
<ul>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>123</li>
<li>132</li>
</ul>
</body>
</html>
</html>

View File

@ -1,89 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>动态订单柱状图</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
canvas {
width: 100% !important;
height: 400px !important;
}
</style>
</head>
<body>
<h2>订单数量柱状图</h2>
<canvas id="orderChart"></canvas>
<script>
// 初始化图表数据
const ctx = document.getElementById('orderChart').getContext('2d');
const chartData = {
labels: [],
datasets: [{
label: '订单数量',
data: [],
backgroundColor: 'rgba(75, 192, 192, 0.6)', // 柱状图的颜色
borderColor: 'rgb(75, 192, 192)',
borderWidth: 1
}]
};
const config = {
type: 'bar', // 设置为柱状图
data: chartData,
options: {
responsive: true,
scales: {
x: {
type: 'category',
position: 'bottom',
},
y: {
beginAtZero: true,
ticks: {
stepSize: 1
}
}
}
}
};
const orderChart = new Chart(ctx, config);
// 获取数据并更新图表
function fetchDataAndUpdateChart() {
fetch('/api/orders-count')
.then(response => response.json())
.then(data => {
// 按照 order_name 分类,汇总 order_count 的数量
let orderNames = {};
data.forEach(order => {
const name = order.order_name;
const count = parseInt(order.order_count);
if (orderNames[name]) {
orderNames[name] += count;
} else {
orderNames[name] = count;
}
});
// 更新图表数据
chartData.labels = Object.keys(orderNames); // 使用订单名称作为X轴标签
chartData.datasets[0].data = Object.values(orderNames); // 使用订单数量作为Y轴数据
orderChart.update();
})
.catch(error => console.error('获取数据失败:', error));
}
// 每5秒更新一次图表数据
setInterval(fetchDataAndUpdateChart, 10000);
// 初始数据加载
fetchDataAndUpdateChart();
</script>
</body>
</html>

View File

@ -1,105 +0,0 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>订单展示</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 0;
padding: 20px;
background-color: #ffffff;
}
h1 {
text-align: center;
color: #333;
}
.container {
max-width: 1200px;
margin: 0 auto;
padding: 20px;
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 20px;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
}
th, td {
padding: 12px;
text-align: left;
border: 1px solid #ddd;
}
th {
background-color: #f2f2f2;
}
tr:nth-child(even) {
background-color: #f9f9f9;
}
.valid {
color: green;
font-weight: bold;
}
.invalid {
color: red;
font-weight: bold;
}
</style>
</head>
<body>
<div class="container">
<h1>订单信息</h1>
<table id="orders-table">
<thead>
<tr>
<th>订单ID</th>
<th>订单分类</th>
<th>订单名称</th>
<th>数量</th>
<th>日期</th>
<th>是否有效</th>
</tr>
</thead>
<tbody>
<!-- 这里将通过 JavaScript 动态填充订单数据 -->
</tbody>
</table>
</div>
<script>
// 定义一个函数用于请求数据并更新表格
function fetchAndUpdateOrders() {
fetch('/api/rawdata') // Flask API 路径
.then(response => response.json())
.then(data => {
const tableBody = document.querySelector('#orders-table tbody');
// 清空现有的表格内容
tableBody.innerHTML = '';
// 填充新的数据
data.forEach(order => {
const row = document.createElement('tr');
row.innerHTML = `
<td>${order.order_id}</td>
<td>${order.order_category}</td>
<td>${order.order_name}</td>
<td>${order.order_quantity}</td>
<td>${order.date}</td>
<td class="${order.is_valid === 'Y' ? 'valid' : 'invalid'}">${order.is_valid === 'Y' ? '有效' : '无效'}</td>
`;
tableBody.appendChild(row);
});
})
.catch(error => {
console.error('获取订单数据失败:', error);
});
}
// 页面加载时,立即调用一次更新数据的函数
fetchAndUpdateOrders();
// 设置轮询:每 5 秒请求一次数据并更新表格
setInterval(fetchAndUpdateOrders, 5000); // 5000 毫秒 = 5 秒
</script>
</body>
</html>

View File

@ -1,91 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>动态订单数量图表</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
canvas {
width: 100% !important;
height: 400px !important;
}
</style>
</head>
<body>
<h2>订单名称数量动态图表</h2>
<canvas id="orderChart"></canvas>
<script>
// 初始化图表
const ctx = document.getElementById('orderChart').getContext('2d');
const chartData = {
labels: [], // X轴标签
datasets: [{
label: '订单数量',
data: [], // Y轴数据
borderColor: 'rgb(75, 192, 192)', // 线条颜色
backgroundColor: 'rgba(75, 192, 192, 0.2)', // 背景填充色
borderWidth: 1
}]
};
const config = {
type: 'bar', // 使用条形图
data: chartData,
options: {
responsive: true,
scales: {
x: {
type: 'category', // X轴使用类别型
position: 'bottom',
},
y: {
beginAtZero: true, // Y轴从0开始
ticks: {
stepSize: 1
}
}
}
}
};
const orderChart = new Chart(ctx, config);
// 获取数据并更新图表
function fetchDataAndUpdateChart() {
fetch('/api/stream/ordernamecount')
.then(response => response.json())
.then(data => {
// 统计每个订单名称的数量
let orderNames = {};
data.forEach(order => {
const name = order.order_name; // 获取订单名称
const count = order.order_name_count; // 获取订单数量
// 累加相同订单名称的数量
if (orderNames[name]) {
orderNames[name] += count;
} else {
orderNames[name] = count;
}
});
// 更新图表数据
chartData.labels = Object.keys(orderNames); // X轴标签为订单名称
chartData.datasets[0].data = Object.values(orderNames); // Y轴数据为订单数量
// 更新图表
orderChart.update();
})
.catch(error => console.error('获取数据失败:', error));
}
// 每5秒更新一次数据
setInterval(fetchDataAndUpdateChart, 10000);
// 初始数据加载
fetchDataAndUpdateChart();
</script>
</body>
</html>

View File

@ -1,100 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>动态订单数量图表</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
canvas {
width: 100% !important;
height: 400px !important;
}
</style>
</head>
<body>
<h2>订单数量动态图表</h2>
<canvas id="orderChart"></canvas>
<script>
// 初始化图表
const ctx = document.getElementById('orderChart').getContext('2d');
const chartData = {
labels: [], // X轴标签
datasets: [{
label: '订单数量',
data: [], // Y轴数据
borderColor: 'rgb(75, 192, 192)', // 线条颜色
backgroundColor: 'rgba(75, 192, 192, 0.2)', // 背景填充色
borderWidth: 1
}]
};
const config = {
type: 'line', // 使用折线图
data: chartData,
options: {
responsive: true,
plugins: {
legend: {
display: true, // 显示图例
}
},
scales: {
x: {
type: 'category', // X轴使用类别型
position: 'bottom',
},
y: {
beginAtZero: true, // Y轴从0开始
ticks: {
stepSize: 1
}
}
}
}
};
const orderChart = new Chart(ctx, config);
// 获取数据并更新图表
function fetchDataAndUpdateChart() {
fetch('/api/stream/ordersummary')
.then(response => response.json())
.then(data => {
console.log('Received data:', data);
// 统计每个时间点的订单数量
let orderCounts = {};
data.forEach(order => {
const status = order.status; // 获取订单时间
const count = order.count; // 获取订单数量
// 按时间统计数量
if (orderCounts[status]) {
orderCounts[status] += count;
} else {
orderCounts[status] = count;
}
});
console.log('Order counts:', orderCounts);
// 更新图表数据
chartData.labels = Object.keys(orderCounts); // 设置X轴标签为时间
chartData.datasets[0].data = Object.values(orderCounts); // 设置Y轴数据为订单数量
// 更新图表
orderChart.update();
})
.catch(error => console.error('获取数据失败:', error));
}
// 每10秒更新一次数据
setInterval(fetchDataAndUpdateChart, 10000);
// 初始数据加载
fetchDataAndUpdateChart();
</script>
</body>
</html>

View File

@ -1,100 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>订单数量随时间变化图表</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
canvas {
width: 100% !important;
height: 400px !important;
}
</style>
</head>
<body>
<h2>订单数量随时间变化图表</h2>
<canvas id="orderChart"></canvas>
<script>
// 初始化图表
const ctx = document.getElementById('orderChart').getContext('2d');
const chartData = {
labels: [], // 时间戳标签
datasets: [{
label: '累计订单数量',
data: [], // 累计的订单数量
borderColor: 'rgb(75, 192, 192)', // 线条颜色
backgroundColor: 'rgba(75, 192, 192, 0.2)', // 背景填充色
borderWidth: 1,
fill: false
}]
};
const config = {
type: 'line', // 使用折线图
data: chartData,
options: {
responsive: true,
scales: {
x: {
type: 'linear', // X轴为线性类型
position: 'bottom',
title: {
display: true,
text: '时间 (秒)'
}
},
y: {
beginAtZero: true, // Y轴从0开始
ticks: {
stepSize: 1
},
title: {
display: true,
text: '订单数量'
}
}
}
}
};
const orderChart = new Chart(ctx, config);
// 获取数据并更新图表
function fetchDataAndUpdateChart() {
fetch('/api/stream/summary')
.then(response => response.json())
.then(data => {
// 累计订单数量
let cumulativeCount = 0;
let labels = [];
let counts = [];
data.forEach(order => {
const status = order.status; // 获取订单时间
const count = order.count; // 获取订单数量
cumulativeCount += count; // 累加数量
labels.push(status); // 保存时间戳
counts.push(cumulativeCount); // 保存累计数量
});
// 更新图表数据
chartData.labels = labels; // 设置时间戳为X轴标签
chartData.datasets[0].data = counts; // 设置累计的订单数量为Y轴数据
// 更新图表
orderChart.update();
})
.catch(error => console.error('获取数据失败:', error));
}
// 每5秒更新一次数据
setInterval(fetchDataAndUpdateChart, 10000);
// 初始数据加载
fetchDataAndUpdateChart();
</script>
</body>
</html>