manager: Add first version of the techtree manager

The techtree manager is a CI tool that derives the techtree graph from
the forgejo issues in this repository.  It then adds graph
visualizations to each issue and the wiki, to give an overview of the
tree.
This commit is contained in:
Rahix 2025-05-22 09:48:01 +02:00
parent 8ab73f4a4a
commit 870e54e263
9 changed files with 2768 additions and 0 deletions

1
techtree-manager/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target/

1986
techtree-manager/Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,18 @@
[package]
name = "techtree-manager"
version = "0.1.0"
edition = "2024"
[dependencies]
anyhow = "1.0.98"
base64 = "0.22.1"
chrono = "0.4.41"
env_logger = { version = "0.11.8", default-features = false, features = ["auto-color", "color", "humantime"] }
forgejo-api = { git = "https://git.fa-fo.de/rahix/forgejo-api.git", rev = "a3f6452cfe774898a89ac66be393e5205f5e12b7" }
log = "0.4.27"
petgraph = "0.8.1"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
sha256 = "1.6.0"
tokio = { version = "1.45.0", features = ["full"] }
url = "2.5.4"

View file

@ -0,0 +1,132 @@
use anyhow::Context as _;
use std::collections::BTreeMap;
/// Read all issues to generate the full techtree
pub async fn collect_tree(
forgejo: &forgejo_api::Forgejo,
meta: &crate::event_meta::IssueEventMeta,
) -> anyhow::Result<crate::tree::Tree> {
let issues = forgejo
.issue_list_issues(
&meta.issue.repository.owner,
&meta.issue.repository.name,
forgejo_api::structs::IssueListIssuesQuery {
// We also want the closed issues
state: Some(forgejo_api::structs::IssueListIssuesQueryState::All),
// No pagination
limit: None,
// Only issues
r#type: Some(forgejo_api::structs::IssueListIssuesQueryType::Issues),
..Default::default()
},
)
.await
.context("Failed fetching issue list")?;
let mut tree = crate::tree::Tree::new();
let mut issue_numbers = Vec::new();
for issue in issues.iter() {
let element = match element_from_issue(issue) {
Ok(el) => el,
Err(e) => {
let maybe_number = issue
.number
.map(|n| n.to_string())
.unwrap_or("<unknown>".to_owned());
log::warn!("Failed processing issue #{maybe_number}: {e:?}");
continue;
}
};
issue_numbers.push(element.issue_number);
tree.add_element(element);
}
for issue in issue_numbers.into_iter() {
let dependencies = forgejo
.issue_list_issue_dependencies(
&meta.issue.repository.owner,
&meta.issue.repository.name,
&issue.to_string(),
forgejo_api::structs::IssueListIssueDependenciesQuery {
limit: None,
..Default::default()
},
)
.await
.with_context(|| {
format!(
"Failed to fetch issue dependencies for #{}",
meta.issue.number
)
})?;
for dep in dependencies {
let dep_number = dep.number.context("Missing issue number in dependency")?;
if !tree.find_element_by_issue_number(dep_number).is_some() {
log::warn!("Found dependency from #{issue} on non-tracked issue #{dep_number}!");
}
tree.add_dependency_by_issue_number(issue, dep_number);
}
}
Ok(tree)
}
fn element_from_issue(issue: &forgejo_api::structs::Issue) -> anyhow::Result<crate::tree::Element> {
let issue_number = issue.number.context("Missing issue number")?;
let description = issue
.title
.as_deref()
.context("Issue is missing a title")?
.to_owned();
let ty_labels: Vec<_> = issue
.labels
.as_ref()
.context("Issue does not have any labels")?
.iter()
.filter_map(|l| l.name.as_deref())
.filter(|l| l.starts_with("ty/"))
.collect();
if ty_labels.len() == 0 {
anyhow::bail!("Issue #{issue_number} has no type label!");
}
if ty_labels.len() > 1 {
anyhow::bail!("Issue #{issue_number} has more than one type label!");
}
let ty = match *ty_labels.first().unwrap() {
"ty/equipment" => crate::tree::ElementType::Equipment,
"ty/process" => crate::tree::ElementType::Process,
"ty/knowledge" => crate::tree::ElementType::Knowledge,
t => anyhow::bail!("Unknown element type for issue #{issue_number}: {t:?}"),
};
let status = match issue.state.context("Missing issue state")? {
forgejo_api::structs::StateType::Open => {
if issue.assignee.is_some()
|| issue
.assignees
.as_ref()
.map(|v| v.len() > 0)
.unwrap_or(false)
{
crate::tree::ElementStatus::Assigned
} else {
crate::tree::ElementStatus::Missing
}
}
forgejo_api::structs::StateType::Closed => crate::tree::ElementStatus::Completed,
};
Ok(crate::tree::Element {
issue_number,
description,
ty,
status,
})
}

View file

@ -0,0 +1,57 @@
use anyhow::Context as _;
#[derive(serde::Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct IssueEventMeta {
pub action: IssueAction,
pub issue: IssueMeta,
}
#[derive(serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum IssueAction {
Opened,
Reopened,
Closed,
Assigned,
Unassigned,
Edited,
#[serde(rename = "label_updated")]
LabelUpdated,
Labeled,
#[serde(rename = "label_cleared")]
LabelCleared,
Unlabeled,
}
#[derive(serde::Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct IssueMeta {
pub number: u64,
pub repository: RepoMeta,
}
#[derive(serde::Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct RepoMeta {
pub name: String,
pub owner: String,
}
pub fn get_issue_event_meta_from_env() -> anyhow::Result<IssueEventMeta> {
let path = std::env::var_os("GITHUB_EVENT_PATH")
.context("Could not get event description file path (GITHUB_EVENT_PATH)")?;
let f = std::fs::File::open(path).context("Could not open GITHUB_EVENT_PATH file")?;
let meta: IssueEventMeta = serde_json::de::from_reader(f).context("Failed to parse")?;
Ok(meta)
}
pub fn fake() -> IssueEventMeta {
IssueEventMeta {
action: IssueAction::Edited,
issue: IssueMeta {
number: 1337,
repository: RepoMeta {
name: "techtree-poc".to_owned(),
owner: "rahix".to_owned(),
},
},
}
}

View file

@ -0,0 +1,82 @@
use anyhow::Context as _;
pub type CommentId = u64;
pub struct BotCommentInfo {
pub body: String,
pub id: CommentId,
}
pub async fn make_bot_comment(
forgejo: &forgejo_api::Forgejo,
meta: &crate::event_meta::IssueEventMeta,
issue_number: u64,
) -> anyhow::Result<CommentId> {
let initial_message =
"_Please be patient, this issue is currently being integrated into the techtree..._";
let res = forgejo
.issue_create_comment(
&meta.issue.repository.owner,
&meta.issue.repository.name,
issue_number,
forgejo_api::structs::CreateIssueCommentOption {
body: initial_message.to_owned(),
updated_at: None,
},
)
.await?;
Ok(res.id.unwrap())
}
pub async fn find_bot_comment(
forgejo: &forgejo_api::Forgejo,
meta: &crate::event_meta::RepoMeta,
issue_number: u64,
) -> anyhow::Result<Option<BotCommentInfo>> {
let mut comments = forgejo
.issue_get_comments(
&meta.owner,
&meta.name,
issue_number,
forgejo_api::structs::IssueGetCommentsQuery {
..Default::default()
},
)
.await
.context("Failed fetching comments for issue")?;
comments.sort_by_key(|comment| comment.created_at);
let maybe_bot_comment = comments
.iter()
.rev()
.find(|comment| comment.user.as_ref().unwrap().id == Some(-2));
Ok(maybe_bot_comment.map(|c| BotCommentInfo {
body: c.body.clone().unwrap_or("".to_owned()),
id: c.id.unwrap(),
}))
}
pub async fn update_bot_comment(
forgejo: &forgejo_api::Forgejo,
meta: &crate::event_meta::RepoMeta,
id: CommentId,
new_body: String,
) -> anyhow::Result<()> {
forgejo
.issue_edit_comment(
&meta.owner,
&meta.name,
id,
forgejo_api::structs::EditIssueCommentOption {
body: new_body,
updated_at: None,
},
)
.await
.context("Failed to update comment body")?;
Ok(())
}

View file

@ -0,0 +1,125 @@
use anyhow::Context as _;
use forgejo_api::Forgejo;
mod collect;
mod event_meta;
mod issue;
mod tree;
mod wiki;
async fn run() -> anyhow::Result<()> {
let meta = if std::env::var("TECHTREE_FAKE").ok().is_some() {
log::warn!("Fake tree!");
event_meta::fake()
} else {
event_meta::get_issue_event_meta_from_env().context("Failed reading issue event data")?
};
log::info!(
"Running due to event \"{:?}\" on issue #{} ...",
meta.action,
meta.issue.number
);
let timestamp = chrono::Local::now().format("%Y-%m-%d %H:%M:%S");
log::info!("Timestamp of this run is {timestamp}");
let token =
std::env::var("GITHUB_TOKEN").context("Failed accessing GITHUB_TOKEN auth token")?;
let auth = forgejo_api::Auth::Token(&token);
let server_url = url::Url::parse(
&std::env::var("GITHUB_SERVER_URL")
.context("Failed reading GITHUB_SERVER_URL server url")?,
)
.context("Failed parsing GITHUB_SERVER_URL as a url")?;
let forgejo = Forgejo::new(auth, server_url).context("Could not create API access object")?;
let new_comment_id = if meta.action == event_meta::IssueAction::Opened {
let res = issue::make_bot_comment(&forgejo, &meta, meta.issue.number).await;
match res {
Ok(id) => Some(id),
Err(e) => {
log::warn!(
"Error while creating the informational comment on issue #{}:\n{e:?}",
meta.issue.number
);
None
}
}
} else {
None
};
let tree = collect::collect_tree(&forgejo, &meta)
.await
.context("Failed to collect the techtree from issue metadata")?;
let mermaid = tree.to_mermaid();
let wiki_text = format!(
r##"This page is automatically updated to show the latest and greatest FAFO techtree:
```mermaid
{mermaid}
```
"##
);
log::info!("Updating the wiki overview...");
wiki::update_wiki_overview(&forgejo, &meta.issue.repository, timestamp.to_string(), wiki_text)
.await
.context("Failed to update the techtree wiki page")?;
'issues: for issue in tree.iter_issues() {
let subtree = tree.subtree_for_issue(issue).unwrap();
let hash = subtree.stable_hash();
let comment_id = if new_comment_id.is_some() && issue == meta.issue.number {
new_comment_id.unwrap()
} else {
let bot_comment = issue::find_bot_comment(&forgejo, &meta.issue.repository, issue)
.await
.with_context(|| format!("Failed searching for bot comment for issue #{issue}"))?;
if let Some(bot_comment) = bot_comment {
if bot_comment.body.contains(&hash) {
log::info!("Issue #{issue} is up-to-date, not editing comment.");
continue 'issues;
}
bot_comment.id
} else {
log::warn!("Missing bot comment in issue #{issue}");
issue::make_bot_comment(&forgejo, &meta, issue)
.await
.with_context(|| {
format!("Failed to create a retrospective bot comment on issue #{issue}")
})?
}
};
let mermaid = subtree.to_mermaid();
let full_text = format!(
r##"## Partial Techtree
```mermaid
{mermaid}
```
<small>Digest: {hash}; Last Updated: {timestamp}</small>"##
);
log::info!("Updating bot comment in issue #{issue} ...");
issue::update_bot_comment(&forgejo, &meta.issue.repository, comment_id, full_text)
.await
.with_context(|| format!("Failed to update the bot comment in issue #{issue}"))?;
}
Ok(())
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
run().await
}

View file

@ -0,0 +1,343 @@
use std::collections::BTreeMap;
use petgraph::visit::EdgeRef as _;
/// Element in the techtree
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize)]
pub struct Element {
/// Issue associated with this element
pub issue_number: u64,
/// Description of this element
pub description: String,
/// Type of this element
pub ty: ElementType,
/// Completion status of this element.
pub status: ElementStatus,
}
impl Element {
fn to_dot_node_attributes(&self, role: Option<SubtreeElementRole>) -> String {
let Element {
issue_number,
description,
ty,
status,
} = self;
let mut attributes = Vec::new();
attributes.push(format!(
r##"label = "{{{{#{issue_number} | {status}}}|{ty}|{description}}}""##
));
attributes.push(r#"shape = "record""#.to_owned());
let color = match (role, status) {
(Some(SubtreeElementRole::ElementOfInterest), _) => "black",
(Some(SubtreeElementRole::Dependant), _) => "gray",
(_, ElementStatus::Missing) => "darkred",
(_, ElementStatus::Assigned) => "orange",
(_, ElementStatus::Completed) => "darkgreen",
};
attributes.push(format!(r#"color = "{color}""#));
attributes.push(format!(r#"fontcolor = "{color}""#));
attributes.join(", ")
}
fn to_mermaid_node(&self, index: ElementIndex, role: Option<SubtreeElementRole>) -> String {
let Element {
issue_number,
description,
ty,
status,
} = self;
let label = format!(r##"<a href='/rahix/techtree-poc/issues/{issue_number}' target='_blank'>#{issue_number}</a> | {status}\n<i>{ty}</i>\n<b>{description}</b>"##);
let class = match (role, status) {
(Some(SubtreeElementRole::ElementOfInterest), _) => "eoi",
(Some(SubtreeElementRole::Dependant), _) => "dependant",
(_, ElementStatus::Missing) => "dep_missing",
(_, ElementStatus::Assigned) => "dep_assigned",
(_, ElementStatus::Completed) => "dep_completed",
};
format!(
" {index}:::{class}\n {index}[\"{label}\"]",
index = index.index(),
)
}
}
fn mermaid_classes() -> String {
r##"
classDef eoi fill:#fff, stroke:#000;
classDef dependant fill:#fff, stroke:#888, color:#888;
classDef dep_missing fill:#fcc, stroke:#800;
classDef dep_assigned fill:#ffa, stroke:#a50;
classDef dep_completed fill:#afa, stroke:#080;
"##
.to_owned()
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, serde::Serialize)]
pub enum ElementType {
Equipment,
Process,
Knowledge,
}
impl std::fmt::Display for ElementType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
ElementType::Equipment => "Equipment",
ElementType::Process => "Process",
ElementType::Knowledge => "Knowledge",
})
}
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, serde::Serialize)]
pub enum ElementStatus {
Missing,
Assigned,
Completed,
}
impl std::fmt::Display for ElementStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
ElementStatus::Missing => "MISSING",
ElementStatus::Assigned => "ASSIGNED",
ElementStatus::Completed => "COMPLETED",
})
}
}
pub type ElementIndex = petgraph::graph::NodeIndex;
pub struct Tree {
graph: petgraph::Graph<Element, ()>,
issue_map: BTreeMap<u64, ElementIndex>,
}
impl Tree {
pub fn new() -> Self {
Self {
graph: petgraph::Graph::new(),
issue_map: BTreeMap::new(),
}
}
pub fn add_element(&mut self, element: Element) {
let issue_number = element.issue_number;
assert!(!self.issue_map.contains_key(&issue_number));
let idx = self.graph.add_node(element);
self.issue_map.insert(issue_number, idx);
}
pub fn add_dependency_by_issue_number(&mut self, dependant: u64, dependency: u64) {
let a = self.find_element_by_issue_number(dependant).unwrap();
let b = self.find_element_by_issue_number(dependency).unwrap();
self.graph.add_edge(a, b, ());
}
pub fn find_element_by_issue_number(&self, issue_number: u64) -> Option<ElementIndex> {
self.issue_map.get(&issue_number).copied()
}
pub fn subtree_for_issue<'a>(&'a self, issue_number: u64) -> Option<Subtree<'a>> {
Some(Subtree::new_for_element(
self,
self.find_element_by_issue_number(issue_number)?,
))
}
pub fn iter_issues<'a>(&'a self) -> impl Iterator<Item = u64> + 'a {
self.issue_map.keys().copied()
}
pub fn subtree_for_element<'a>(&'a self, element: ElementIndex) -> Subtree<'a> {
Subtree::new_for_element(self, element)
}
pub fn to_dot(&self) -> String {
let dot = petgraph::dot::Dot::with_attr_getters(
&self.graph,
&[
petgraph::dot::Config::EdgeNoLabel,
petgraph::dot::Config::NodeNoLabel,
petgraph::dot::Config::RankDir(petgraph::dot::RankDir::BT),
],
&|_g, _edge_id| "".to_string(),
&|_g, (_, element)| element.to_dot_node_attributes(None),
);
format!("{:?}", dot)
}
pub fn to_mermaid(&self) -> String {
let mut mermaid = String::new();
mermaid.push_str("flowchart BT\n");
mermaid.push_str(&mermaid_classes());
for index in self.graph.node_indices() {
mermaid.push_str(&self.graph[index].to_mermaid_node(index, None));
mermaid.push_str("\n");
}
for edge in self.graph.edge_references() {
mermaid.push_str(&format!(
" {} --> {}\n",
edge.source().index(),
edge.target().index()
));
}
mermaid
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize)]
pub struct SubtreeElement<'a> {
element: &'a Element,
role: SubtreeElementRole,
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, serde::Serialize)]
pub enum SubtreeElementRole {
ElementOfInterest,
Dependency,
Dependant,
}
pub struct Subtree<'a> {
original: &'a Tree,
graph: petgraph::Graph<SubtreeElement<'a>, ()>,
index_map: BTreeMap<ElementIndex, ElementIndex>,
}
impl<'a> Subtree<'a> {
pub fn new_for_element(tree: &'a Tree, element: ElementIndex) -> Subtree<'a> {
let mut this = Self {
original: tree,
graph: petgraph::Graph::new(),
index_map: BTreeMap::new(),
};
let graph = &this.original.graph;
this.add_node_if_missing(element, SubtreeElementRole::ElementOfInterest);
let mut dfs = petgraph::visit::Dfs::new(&graph, element);
while let Some(idx) = dfs.next(&graph) {
this.add_node_if_missing(idx, SubtreeElementRole::Dependency);
}
for idx in this.index_map.keys().copied() {
for neighbor in graph.neighbors_directed(idx, petgraph::Direction::Outgoing) {
if !this.index_map.contains_key(&neighbor) {
log::warn!(
"Probably missed a dependency from #{from} to #{to} while generating subtree for #{el}",
el = graph[element].issue_number,
from = graph[idx].issue_number,
to = graph[neighbor].issue_number
);
continue;
}
this.graph
.add_edge(this.index_map[&idx], this.index_map[&neighbor], ());
}
}
for neighbor in graph.neighbors_directed(element, petgraph::Direction::Incoming) {
if this.index_map.contains_key(&neighbor) {
log::warn!(
"Already found #{from} in dependencies of #{to}, but it should have been the other way around?!",
from = graph[neighbor].issue_number,
to = graph[element].issue_number
);
} else {
this.add_node_if_missing(neighbor, SubtreeElementRole::Dependant);
}
this.graph
.add_edge(this.index_map[&neighbor], this.index_map[&element], ());
}
this
}
fn add_node_if_missing(&mut self, index: ElementIndex, role: SubtreeElementRole) -> bool {
if self.index_map.contains_key(&index) {
debug_assert!(self.graph.node_weight(self.index_map[&index]).is_some());
return false;
}
let subtree_element = SubtreeElement {
element: &self.original.graph[index],
role,
};
let new_idx = self.graph.add_node(subtree_element);
self.index_map.insert(index, new_idx);
return true;
}
pub fn to_dot(&self) -> String {
let dot = petgraph::dot::Dot::with_attr_getters(
&self.graph,
&[
petgraph::dot::Config::EdgeNoLabel,
petgraph::dot::Config::NodeNoLabel,
petgraph::dot::Config::RankDir(petgraph::dot::RankDir::BT),
],
&|_g, _edge_id| "".to_string(),
&|_g, (_, element)| element.element.to_dot_node_attributes(Some(element.role)),
);
format!("{:?}", dot)
}
pub fn to_mermaid(&self) -> String {
let mut mermaid = String::new();
mermaid.push_str("flowchart BT\n");
mermaid.push_str(&mermaid_classes());
for index in self.graph.node_indices() {
let node = &self.graph[index];
mermaid.push_str(&node.element.to_mermaid_node(index, Some(node.role)));
mermaid.push_str("\n");
}
for edge in self.graph.edge_references() {
mermaid.push_str(&format!(
" {} --> {}\n",
edge.source().index(),
edge.target().index()
));
}
mermaid
}
pub fn stable_hash(&self) -> String {
let mut nodes: Vec<_> = self.graph.node_weights().collect();
nodes.sort_by_key(|n| n.element.issue_number);
let mut edges: Vec<_> = self
.graph
.edge_references()
.map(|edge| {
(
self.graph[edge.source()].element.issue_number,
self.graph[edge.target()].element.issue_number,
)
})
.collect();
edges.sort();
let json_data = serde_json::ser::to_string(&(nodes, edges)).unwrap();
sha256::digest(json_data)
}
}

View file

@ -0,0 +1,24 @@
use anyhow::Context as _;
pub async fn update_wiki_overview(
forgejo: &forgejo_api::Forgejo,
meta: &crate::event_meta::RepoMeta,
timestamp: String,
new_body: String,
) -> anyhow::Result<()> {
// TODO: Figure out why we get a 404 when the edit was successfull...
let _ = forgejo
.repo_edit_wiki_page(
&meta.owner,
&meta.name,
"Home",
forgejo_api::structs::CreateWikiPageOptions {
content_base64: Some(base64::encode(new_body.as_bytes())),
message: Some(format!("Updated to latest model at {timestamp}")),
title: Some("Home".to_owned()),
},
)
.await
.context("Failed editing the wiki page");
Ok(())
}