Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* @acompany-develop/platform
16 changes: 16 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Description of changes・変更内容の説明

# Ticket link・チケットリンク
https://www.notion.so/acompany-ac/<チケットID>

# Testing method・実施したテスト方法 (CI/Manual)
- UTs :
- E2E test :
- Integration test :


# Remarks・備考


# Review guidelines
[Generic PR Review guidelines・一般的な PR レビューのガイドライン](https://www.notion.so/acompany-ac/Generic-PR-Review-guidelines-PR-13b269d8558680f6918bdccee7812efb?pvs=4)
31 changes: 31 additions & 0 deletions .github/workflows/ai-review.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: "Run ai-reviewer"

permissions:
pull-requests: write
contents: read

on:
pull_request:
types: [opened, reopened, ready_for_review]
workflow_dispatch:

jobs:
run-review:
runs-on: ubuntu-latest
steps:
- name: AI Review Bot
uses: acompany-develop/ai-reviewer@latest
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANGUAGE: |-
Japanese
English
PRIORITY_FILTER: |-
PRIORITY:HIGH
PRIORITY:MEDIUM
PRIORITY:LOW
PRIORITY:POSITIVE
# ref: https://www.notion.so/acompany-ac/Generic-PR-Review-guidelines-PR-13b269d8558680f6918bdccee7812efb
EXTRA_PROMPT: |-
${{ vars.PR_REVIEW_POLICY_FOR_LLM }}
59 changes: 59 additions & 0 deletions .github/workflows/reassign-reviewer.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Re-assign reviewers even after approval
on:
pull_request_review:
types: [submitted]

jobs:
keep_team_reviewers:
runs-on: ubuntu-latest
permissions:
pull-requests: write
contents: read
steps:
- name: Re-assign platform Team After Approval
uses: actions/github-script@v6
with:
github-token: ${{ secrets.WORKFLOW_PAT }}
script: |
const { owner, repo, number } = context.issue;
const review = context.payload.review;

console.log(`Review submitted by ${review.user.login} with state: ${review.state}`);

// Check if this was an approval
if (review.state === 'approved') {
console.log('Review was an approval, checking team reviewers status');

try {
const teamSlug = "platform";

// Re-request review using the correct team slug format
await github.rest.pulls.requestReviewers({
owner,
repo,
pull_number: number,
team_reviewers: [teamSlug]
});

console.log(`Successfully re-requested review from team: ${teamSlug}`);
} catch (error) {
console.error('Error re-requesting team review:');
console.error(`Status: ${error.status}`);
console.error(`Message: ${error.message}`);

// If re-requesting fails, fall back to a comment
try {
await github.rest.issues.createComment({
owner,
repo,
issue_number: number,
body: '✅ This PR has been approved, but other members of @acompany-develop/platform may still want to review it.'
});
console.log('Added comment mentioning the team as fallback');
} catch (commentError) {
console.error('Error adding comment:', commentError.message);
}
}
} else {
console.log('Review was not an approval, no action taken');
}
141 changes: 141 additions & 0 deletions .github/workflows/ruff-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
name: "Ruff Format & Lint"

permissions:
contents: read
pull-requests: write

on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- '**.py'
workflow_dispatch:

jobs:
ruff-check:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.ref }}

- name: Install ruff without running check or format
uses: astral-sh/ruff-action@v3
with:
args: "--version"

- name: Run Ruff Format & Lint Check
id: ruff-format-lint
run: |
lf='\n'
message="## Ruff チェック結果"

# フォーマットチェック
message+="${lf}### フォーマット"
FORMAT_FAILED=0
ruff format --check > format_output.txt 2>&1 || FORMAT_FAILED=1
FORMAT_OUTPUT=$(cat format_output.txt)

echo "$FORMAT_OUTPUT"

# フォーマット結果の処理
if [ "$FORMAT_FAILED" == "1" ]; then
message+="${lf}❌ フォーマットの問題が検出されました。"
message+="${lf}\`\`\`"
message+="${lf}${FORMAT_OUTPUT}"
message+="${lf}\`\`\`"
else
message+="${lf}✅ フォーマットの問題はありませんでした。"
if [ -n "$FORMAT_OUTPUT" ]; then
message+="${lf}\`\`\`"
message+="${lf}${FORMAT_OUTPUT}"
message+="${lf}\`\`\`"
fi
fi

# # Lintチェック
message+="${lf}### Lint"
LINT_FAILED=0
ruff check > lint_output.txt 2>&1 || LINT_FAILED=1
LINT_OUTPUT=$(cat lint_output.txt)

echo "$LINT_OUTPUT"

# Lint結果の処理
if [ "$LINT_FAILED" == "1" ]; then
message+="${lf}❌ Lint の問題が検出されました。"
message+="${lf}\`\`\`"
message+="${lf}${LINT_OUTPUT}"
message+="${lf}\`\`\`"
else
message+="${lf}✅ Lint の問題はありませんでした。"
if [ -n "$LINT_OUTPUT" ]; then
message+="${lf}\`\`\`"
message+="${lf}${LINT_OUTPUT}"
message+="${lf}\`\`\`"
fi
fi

# Base64エンコードして出力
echo "message=$(echo "$message" | base64 -w 0)" >> $GITHUB_OUTPUT
echo "format_failed=$FORMAT_FAILED" >> $GITHUB_OUTPUT
echo "lint_failed=$LINT_FAILED" >> $GITHUB_OUTPUT

- name: Comment PR with Results
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
// Base64デコードして元のメッセージを取得
const base64Message = '${{ steps.ruff-format-lint.outputs.message }}';
const message = Buffer.from(base64Message, 'base64').toString('utf-8');

// 改行を正しく処理するために、\nをリテラル改行に置換
const formattedMessage = message.replace(/\\n/g, '\n');

const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number
});

const botComment = comments.find(comment =>
comment.user.login === 'github-actions[bot]' &&
comment.body.includes('Ruff チェック結果')
);

if (botComment) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: botComment.id,
body: formattedMessage
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: formattedMessage
});
}

- name: Output Results for non-PR events
if: github.event_name != 'pull_request'
run: |
# Base64デコードして表示(printfで改行を処理)
printf "%b" "$(echo "${{ steps.ruff-format-lint.outputs.message }}" | base64 -d)"

- name: Fail if checks failed
if: steps.ruff-format-lint.outputs.format_failed == '1' || steps.ruff-format-lint.outputs.lint_failed == '1'
run: |
if [ "${{ steps.ruff-format-lint.outputs.format_failed }}" == "1" ]; then
echo "::error::フォーマットチェックが失敗しました"
fi
if [ "${{ steps.ruff-format-lint.outputs.lint_failed }}" == "1" ]; then
echo "::error::Lintチェックが失敗しました"
fi
exit 1
146 changes: 146 additions & 0 deletions functions/cross_table/function/handler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import os
import sys
import traceback
from datetime import datetime

sys.path.insert(0, "/work/function/packages") # functionが依存するパッケージのパス

WORK_DIR = "/work"
INPUT_A_PATH = f"{WORK_DIR}/inputs/input_1"
INPUT_B_PATH = f"{WORK_DIR}/inputs/input_2"
OUTPUT_A_PATH = f"{WORK_DIR}/outputs/output_1"
OUTPUT_B_PATH = f"{WORK_DIR}/outputs/output_2"
DOWNLOAD_DIR = "downloads/"

THRESHOLD = 2 # 集計数がこの値未満の行は出力されない


def print_log(msg):
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
with open(os.path.join(DOWNLOAD_DIR, "app.log"), "a") as log_file:
log_file.write(f"[{current_time}]:[handler.py]: {msg}\n")
os.makedirs(OUTPUT_A_PATH, exist_ok=True)
with open(os.path.join(OUTPUT_A_PATH, "app.log"), "a") as log_file:
log_file.write(f"[{current_time}]:[handler.py]: {msg}\n")
os.makedirs(OUTPUT_B_PATH, exist_ok=True)
with open(os.path.join(OUTPUT_B_PATH, "app.log"), "a") as log_file:
log_file.write(f"[{current_time}]:[handler.py]: {msg}\n")
except Exception:
Comment thread
tak-ka3 marked this conversation as resolved.
pass # 出力ディレクトリへの書き込みに失敗しても続行


# メモリ使用量と実行時間を計測するための関数
def get_memory_usage():
"""現在のメモリ使用量を取得"""
import psutil

process = psutil.Process(os.getpid())
memory_info = process.memory_info()
return memory_info.rss / 1024 / 1024 # MB単位


def print_memory_usage(stage_name):
"""メモリ使用量を表示"""
try:
memory_mb = get_memory_usage()
print_log(f"[{stage_name}] メモリ使用量: {memory_mb:.2f} MB")
except Exception as e:
print_log(f"Failed to get memory usage: {e.__class__.__name__}")
return 0


def run():
try:
print_memory_usage("開始時")
print_log("handler.run: Started.")

import polars as pl

print_log("handler.run: Imported successfully.")

# 入力データを読み込む
lf_a = pl.scan_csv(os.path.join(INPUT_A_PATH, "input_a.csv"))
print_log("handler.run: Read input_a.csv successfully.")
print_memory_usage("input_a.csv読み込み後")
lf_b = pl.scan_csv(os.path.join(INPUT_B_PATH, "input_b.csv"))
print_log("handler.run: Read input_b.csv successfully.")
print_memory_usage("input_b.csv読み込み後")

# キー列を特定
key_a = lf_a.columns[0]
key_b = lf_b.columns[0]

# 2. Join前のリネーム処理
# dataset_a の列名をリネーム (id以外)
cols_to_rename_a = [col for col in lf_a.columns if col != key_a]
rename_map_a = {col: f"0:{col}" for col in cols_to_rename_a}
lf_a_renamed = lf_a.rename(rename_map_a)
print_memory_usage("0_列名リネーム後")

# dataset_b の列名をリネーム (id以外)
cols_to_rename_b = [col for col in lf_b.columns if col != key_b]
rename_map_b = {col: f"1:{col}" for col in cols_to_rename_b}
lf_b_renamed = lf_b.rename(rename_map_b)
print_memory_usage("1_列名リネーム後")
print_log("handler.run: Renamed columns successfully.")

# 3. リネーム済みのLazyFrameをJoin
lf_joined = lf_a_renamed.join(
lf_b_renamed, left_on=key_a, right_on=key_b, how="inner"
)
print_log("handler.run: Merged successfully with leftmost columns.")
print_memory_usage("Join後")

# 4. 全Attribute列でGroup By & Count
# id以外の全ての列(a_...とb_...)をグループ化のキーに指定
attribute_cols = [
col for col in lf_joined.columns if col != key_a and col != key_b
]

lf_summary = lf_joined.group_by(attribute_cols).agg(
pl.count().alias("number_of_rows")
)
print_memory_usage("Group By & Count後")

# 5. 列の整形
# number_of_rows を先頭に持ってくる
# 列名をソートしてから指定
sorted_cols = ["number_of_rows"] + sorted(attribute_cols)
lf_final = lf_summary.select(sorted_cols)

filtered = lf_final.filter(pl.col("number_of_rows") >= THRESHOLD)
print_log(f"handler.run: Filtered successfully with threshold {THRESHOLD}.")
print_memory_usage("Filter後")

# 計算を実行して結果を表示
final_result = filtered.collect(streaming=True)
print_memory_usage("Collect後")

# CSV形式で出力
try:
os.makedirs(OUTPUT_A_PATH, exist_ok=True)
os.makedirs(OUTPUT_B_PATH, exist_ok=True)

final_result.write_csv(os.path.join(OUTPUT_A_PATH, "output.csv"))
print_log("handler.run: Saved a's output.csv successfully.")
print_memory_usage("a's output.csv保存後")
final_result.write_csv(os.path.join(OUTPUT_B_PATH, "output.csv"))
print_log("handler.run: Saved b's output.csv successfully.")
print_memory_usage("b's output.csv保存後")
except Exception as e:
print_log(f"handler.run: Error saving results: {str(e)}")

print_log("handler.run: DONE.")
print_memory_usage("終了時")

except BaseException as e:
Comment thread
tak-ka3 marked this conversation as resolved.
print_log(f"handler.run: ERROR: {str(e)}")
try:
os.makedirs(DOWNLOAD_DIR, exist_ok=True)
with open(os.path.join(DOWNLOAD_DIR, "error.log"), "w") as error_file:
traceback.print_exc(file=error_file)
except Exception:
pass # エラーログの書き込みに失敗しても続行
raise e
Loading