August 2022 - gerrit-log - lists.osmocom.org

Change in osmocom-bb[master]: trxcon: l1sched: ensure n_bits_total is always initialized

by fixeria

Attention is currently required from: pespin. fixeria has posted comments on this change. ( https://gerrit.osmocom.org/c/osmocom-bb/+/28866 ) Change subject: trxcon: l1sched: ensure n_bits_total is always initialized ...................................................................... Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmocom-bb/+/28866 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmocom-bb Gerrit-Branch: master Gerrit-Change-Id: I590cfe55365b7ad021a3d0925a0f1ea136e67125 Gerrit-Change-Number: 28866 Gerrit-PatchSet: 1 Gerrit-Owner: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:32:09 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmo-ttcn3-hacks[master]: WIP

by pespin

pespin has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/28883 ) Change subject: WIP ...................................................................... WIP Change-Id: Ifcdcddc7dccb5439126a5fa29bb540669ed25908 --- M cbc/CBC_Tests.ttcn M cbc/MME_ConnectionHandler.ttcn M library/sbcap/SBC_AP_Templates.ttcn 3 files changed, 105 insertions(+), 2 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/83/28883/1 diff --git a/cbc/CBC_Tests.ttcn b/cbc/CBC_Tests.ttcn index 76fc3be..f39b487 100644 --- a/cbc/CBC_Tests.ttcn +++ b/cbc/CBC_Tests.ttcn @@ -115,7 +115,9 @@ cbc_host := cbc_host, cbc_sbcap_port := cbc_sbcap_port, start_fn := refers(f_MME_ConnHdlr_start_fn_void), - exp_cbs_msg := omit + exp_cbs_msg := omit, + write_replace_warning_ind_cause := omit, + bcast_cell_id_list := omit }; return pars; } @@ -298,6 +300,12 @@ private function f_mme_create_and_delete() runs on MME_ConnHdlr { f_sbcap_handle_write_replace_warn_req(g_pars.exp_cbs_msg, 0); + if (ispresent(g_pars.write_replace_warning_ind_cause) and + ispresent(g_pars.bcast_cell_id_list)) { + f_sbcap_tx_write_replace_warn_ind(0, g_pars.exp_cbs_msg, + g_pars.write_replace_warning_ind_cause, + g_pars.bcast_cell_id_list) + } f_sbcap_handle_stop_warn_req(0, g_pars.exp_cbs_msg); } @@ -411,10 +419,32 @@ f_shutdown_helper(); } +/* Create and delete message with MME available. MME reports + * Write-Replace-Warning-Indication and Stop-Warning-Indication to CBC + */ +testcase TC_ecbe_create_delete_mme_indication() runs on test_CT { + var template (value) CellId_Broadcast_List bcast_cell_id_li; + var template (value) CBS_Message msg := t_CBSmsg(48, 16752); + + f_init(num_bsc := 0, num_mme := 1); + + bcast_cell_id_li := { + ts_SBCAP_CellId_Broadcast_List_Item(ts_SBCAP_ECGI(f_enc_mcc_mnc('901'H, '70'H), 1234)) + }; + g_pars_MME[0].start_fn := refers(f_mme_create_and_delete); + g_pars_MME[0].exp_cbs_msg := valueof(msg); + g_pars_MME[0].write_replace_warning_ind_cause := SBC_AP_Cause_message_accepted; + g_pars_MME[0].bcast_cell_id_list := valueof(bcast_cell_id_li); + f_start(); + f_create_and_delete(valueof(msg)); + f_shutdown_helper(); +} + /* Create and delete message with both BSC and MME available */ testcase TC_ecbe_create_delete_bsc_and_mme() runs on test_CT { f_init(num_bsc := 1, num_mme := 1); var template (value) BSSMAP_FIELD_CellIdentificationList cell_list_success; + var template (value) CellId_Broadcast_List bcast_cell_id_li; var template (value) CBS_Message msg := t_CBSmsg(43, 16752); cell_list_success := ts_BSSMAP_CIL_CGI({ @@ -425,8 +455,14 @@ g_pars_BSC[0].start_fn := refers(f_bsc_create_and_delete); g_pars_BSC[0].exp_cbs_msg := valueof(msg); g_pars_BSC[0].cell_list_success := valueof(cell_list_success); + + bcast_cell_id_li := { + ts_SBCAP_CellId_Broadcast_List_Item(ts_SBCAP_ECGI(f_enc_mcc_mnc('901'H, '70'H), 1234)) + }; g_pars_MME[0].start_fn := refers(f_mme_create_and_delete); g_pars_MME[0].exp_cbs_msg := valueof(msg); + g_pars_MME[0].write_replace_warning_ind_cause := SBC_AP_Cause_message_accepted; + g_pars_MME[0].bcast_cell_id_list := valueof(bcast_cell_id_li); f_start(); f_create_and_delete(valueof(msg)); f_shutdown_helper(); @@ -440,6 +476,7 @@ execute( TC_ecbe_create_delete_lac() ); execute( TC_ecbe_create_delete_ci() ); execute( TC_ecbe_create_delete_lai() ); + execute( TC_ecbe_create_delete_mme_indication() ); execute( TC_ecbe_create_delete_bsc_and_mme() ); } diff --git a/cbc/MME_ConnectionHandler.ttcn b/cbc/MME_ConnectionHandler.ttcn index 055aaa3..fec358c 100644 --- a/cbc/MME_ConnectionHandler.ttcn +++ b/cbc/MME_ConnectionHandler.ttcn @@ -35,7 +35,9 @@ charstring cbc_host, integer cbc_sbcap_port, void_fn start_fn, - CBS_Message exp_cbs_msg optional + CBS_Message exp_cbs_msg optional, + SBC_AP_Cause write_replace_warning_ind_cause optional, + CellId_Broadcast_List bcast_cell_id_list optional }; function f_MME_ConnHdlr_main(charstring id, MME_ConnHdlrPars pars) runs on MME_ConnHdlr { @@ -53,6 +55,17 @@ f_SBC_AP_send(tx, idx); } +function f_sbcap_tx_write_replace_warn_ind(integer idx := 0, CBS_Message msg, + SBC_AP_Cause cause, + template (value) CellId_Broadcast_List bcast_cell_id_li) +runs on MME_ConnHdlr { + var template (value) SBC_AP_PDU tx; + tx := ts_SBCAP_WRITE_WARNING_IND(int2bit(msg.msg_id, 16), + int2bit(msg.ser_nr, 16), + cause, bcast_cell_id_li); + f_SBC_AP_send(tx, idx); +} + function f_sbcap_tx_stop_warn_resp(integer idx := 0, CBS_Message msg) runs on MME_ConnHdlr { var template (value) SBC_AP_PDU tx; diff --git a/library/sbcap/SBC_AP_Templates.ttcn b/library/sbcap/SBC_AP_Templates.ttcn index d5f2471..0a9519e 100644 --- a/library/sbcap/SBC_AP_Templates.ttcn +++ b/library/sbcap/SBC_AP_Templates.ttcn @@ -11,6 +11,28 @@ import from SBC_AP_Types all; +/* 3GPP TS 36.413 9.2.1.38 */ +template (value) EUTRAN_CGI ts_SBCAP_ECGI(PLMNidentity plmn_id := '09F107'O, integer cell_id := 1) := { + pLMNidentity := plmn_id, + cell_ID := int2bit(cell_id, 28), + iE_Extensions := omit +} + +/* 3GPP TS 36.413 9.2.1.54 */ +template (value) CellId_Broadcast_List_Item ts_SBCAP_CellId_Broadcast_List_Item( + template (value) EUTRAN_CGI ecgi := ts_SBCAP_ECGI()) := { + eCGI := ecgi, + iE_Extensions := omit +} +template (value) Broadcast_Scheduled_Area_List ts_SBCAP_Broadcast_Scheduled_Area_List( + template (value) CellId_Broadcast_List cell_id_li := {ts_SBCAP_CellId_Broadcast_List_Item()}) +:= { + cellId_Broadcast_List := cell_id_li, + tAI_Broadcast_List := omit, + emergencyAreaID_Broadcast_List := omit, + iE_Extensions := omit +} + template (value) SBC_AP_PDU ts_SBCAP_PWS_RESTART( template (value) Restarted_Cell_List restart_cl, @@ -190,6 +212,37 @@ } } +/* 4.3.4.2.5 WRITE REPLACE WARNING INDICATION */ +template (value) SBC_AP_PDU +ts_SBCAP_WRITE_WARNING_IND(template (value) BIT16 p_msg_id, template (value) BIT16 p_ser_nr, + template (value) SBC_AP_Cause cause := SBC_AP_Cause_message_accepted, + template (value) CellId_Broadcast_List bcast_cell_id_li := {ts_SBCAP_CellId_Broadcast_List_Item()}) := { + initiatingMessage := { + procedureCode := id_Write_Replace_Warning_Indication, + criticality := ignore, + value_ := { + write_Replace_Warning_Indication := { + protocolIEs := { + { + id := SBC_AP_Constants.id_Message_Identifier, + criticality := reject, + value_ := { Message_Identifier := p_msg_id } + }, { + id := SBC_AP_Constants.id_Serial_Number, + criticality := reject, + value_ := { Serial_Number := p_ser_nr } + }, { + id := SBC_AP_Constants.id_Broadcast_Scheduled_Area_List, + criticality := reject, + value_ := { broadcast_Scheduled_Area_List := ts_SBCAP_Broadcast_Scheduled_Area_List(bcast_cell_id_li) } + } + }, + protocolExtensions := omit + } + } + } +} + /* 4.3.4.2.3 STOP WARNING REQUEST */ template (value) SBC_AP_PDU ts_SBCAP_WRITE_WARNING_RESP(template (value) BIT16 p_msg_id, template (value) BIT16 p_ser_nr, -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/28883 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Ifcdcddc7dccb5439126a5fa29bb540669ed25908 Gerrit-Change-Number: 28883 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-MessageType: newchange

2 years, 10 months

1
0
0 0

Change in osmo-ci[master]: scripts/obs: rewrite pushing source pkgs to OBS

by pespin

Attention is currently required from: osmith, laforge, fixeria. pespin has posted comments on this change. ( https://gerrit.osmocom.org/c/osmo-ci/+/28882 ) Change subject: scripts/obs: rewrite pushing source pkgs to OBS ...................................................................... Patch Set 2: Code-Review+1 -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/28882 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 Gerrit-Change-Number: 28882 Gerrit-PatchSet: 2 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: osmith <osmith(a)sysmocom.de> Gerrit-Attention: laforge <laforge(a)osmocom.org> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:29:58 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmo-ci[master]: scripts/obs: rewrite pushing source pkgs to OBS

by osmith

Attention is currently required from: osmith, laforge, fixeria. Hello Jenkins Builder, laforge, pespin, fixeria, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/c/osmo-ci/+/28882 to look at the new patch set (#2). Change subject: scripts/obs: rewrite pushing source pkgs to OBS ...................................................................... scripts/obs: rewrite pushing source pkgs to OBS Harald requested that the OBS scripts should not stop if building one specific source package fails, instead it should keep going and report at the end a non-success exit code. Given that the shell script code has historically grown and became hard to maintain, I decided to rewrite the scripts for implementing this feature. This rewrite solves additional problems: * No full checkout of an OBS project like network:osmocom:latest anymore, with lots of packages that won't get updated (e.g. the uhd package has a uhd-images_3.14.1.1.tar.xz file that is 108 MB). With the old code, developers had to wait minutes during the checkout before the script reaches code that is currently being developed. Now only single packages get checked out right before they get updated. * No need to clone git repositories over and over. With the new code, git repos only get cloned if needed (for latest it is not needed if the remote git tag is the same as the version in OBS). During development, the cloned git repositories are cached. * Output from commands like "git tag -l" is not written to the log unless they failed. This makes the log more readable, which is especially important when a package fails to build, we keep going and need to spot the build error in the middle of the log later on. * No more duplicated code for nightly and latest scripts that worked similar but had slight differences. Also the list of packages is not duplicated for nightly and latest anymore; nightly uses all packages and latest uses packages that have at least one git tag. * Building source packages is decoupled from uploading them. A separate script build_srcpkg.py can be used to just build the deb + rpm spec source packages, without interacting with the OBS server. * The scripts can optionally run in docker with a command-line switch, and this is used by jenkins. This way we don't need to install more dependencies on the host such as rebar3 which is now needed for erlang/osmo_dia2gsup. * Add erlang/osmo_dia2gsup and run its generate_build_dep.sh (SYS#6006) I have done the new implementation in python to make use of argparse and to be able to use try/except and print a trace when building one package fails. Example output: * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_nightly_obs.osmocom.org… * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_latest_obs.osmocom.org/… Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 --- M .gitignore M jobs/osmocom-obs.yml D scripts/common-obs-conflict.sh D scripts/common-obs.sh A scripts/obs/build_srcpkg.py A scripts/obs/data/Dockerfile A scripts/obs/data/Release.key A scripts/obs/data/rpmlintrc A scripts/obs/lib/__init__.py A scripts/obs/lib/config.py A scripts/obs/lib/debian.py A scripts/obs/lib/docker.py A scripts/obs/lib/git.py A scripts/obs/lib/metapkg.py A scripts/obs/lib/osc.py A scripts/obs/lib/rpm_spec.py A scripts/obs/lib/srcpkg.py A scripts/obs/update_obs_project.py D scripts/osmocom-latest-packages.sh D scripts/osmocom-next-packages.sh D scripts/osmocom-nightly-packages.sh 21 files changed, 1,338 insertions(+), 1,016 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/82/28882/2 -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/28882 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 Gerrit-Change-Number: 28882 Gerrit-PatchSet: 2 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: osmith <osmith(a)sysmocom.de> Gerrit-Attention: laforge <laforge(a)osmocom.org> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-MessageType: newpatchset

2 years, 10 months

1
0
0 0

Change in osmocom-bb[master]: trxcon: l1sched: ensure n_bits_total is always initialized

by osmith

Attention is currently required from: fixeria, pespin. osmith has posted comments on this change. ( https://gerrit.osmocom.org/c/osmocom-bb/+/28866 ) Change subject: trxcon: l1sched: ensure n_bits_total is always initialized ...................................................................... Patch Set 1: Code-Review+1 -- To view, visit https://gerrit.osmocom.org/c/osmocom-bb/+/28866 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmocom-bb Gerrit-Branch: master Gerrit-Change-Id: I590cfe55365b7ad021a3d0925a0f1ea136e67125 Gerrit-Change-Number: 28866 Gerrit-PatchSet: 1 Gerrit-Owner: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Attention: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:26:00 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmo-ci[master]: scripts/obs: rewrite pushing source pkgs to OBS

by pespin

Attention is currently required from: osmith, laforge, fixeria. pespin has posted comments on this change. ( https://gerrit.osmocom.org/c/osmo-ci/+/28882 ) Change subject: scripts/obs: rewrite pushing source pkgs to OBS ...................................................................... Patch Set 1: Code-Review+1 (1 comment) Patchset: PS1: Using python for this looks a bit overkill but fine :) -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/28882 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 Gerrit-Change-Number: 28882 Gerrit-PatchSet: 1 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: osmith <osmith(a)sysmocom.de> Gerrit-Attention: laforge <laforge(a)osmocom.org> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:25:37 +0000 Gerrit-HasComments: Yes Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmocom-bb[master]: trxcon: assert *ts is not NULL in l1sched_set_lchans

by osmith

Attention is currently required from: laforge, fixeria. osmith has posted comments on this change. ( https://gerrit.osmocom.org/c/osmocom-bb/+/28810 ) Change subject: trxcon: assert *ts is not NULL in l1sched_set_lchans ...................................................................... Patch Set 3: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmocom-bb/+/28810 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmocom-bb Gerrit-Branch: master Gerrit-Change-Id: I53b40b31f20fc9aefdfcac1406c465d226e5fde0 Gerrit-Change-Number: 28810 Gerrit-PatchSet: 3 Gerrit-Owner: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: laforge <laforge(a)osmocom.org> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:25:08 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmo-hnbgw[master]: tests/ranap_rab_ass: fix potential NULL pointer dereferences

by neels

neels has submitted this change. ( https://gerrit.osmocom.org/c/osmo-hnbgw/+/28837 ) Change subject: tests/ranap_rab_ass: fix potential NULL pointer dereferences ...................................................................... tests/ranap_rab_ass: fix potential NULL pointer dereferences Change-Id: I16fea7b2a8cb1d693e01c91d7633550e2e599ceb Related: CID#275345 --- M tests/ranap_rab_ass/ranap_rab_ass_test.c M tests/ranap_rab_ass/ranap_rab_ass_test.ok 2 files changed, 2 insertions(+), 2 deletions(-) Approvals: Jenkins Builder: Verified neels: Looks good to me, approved diff --git a/tests/ranap_rab_ass/ranap_rab_ass_test.c b/tests/ranap_rab_ass/ranap_rab_ass_test.c index 043b16b..f4e7999 100644 --- a/tests/ranap_rab_ass/ranap_rab_ass_test.c +++ b/tests/ranap_rab_ass/ranap_rab_ass_test.c @@ -61,7 +61,7 @@ OSMO_ASSERT(rc == 0); encoded = ranap_rab_ass_req_encode(&message.msg.raB_AssignmentRequestIEs); - printf("ranap_rab_ass_req_encode %s\n", encoded ? "ok" : "ERROR"); + OSMO_ASSERT(encoded != NULL); printf("INPUT: %s\n", osmo_hexdump_nospc(testvec, sizeof(testvec))); printf("RESULT: %s\n", osmo_hexdump_nospc(encoded->data, encoded->len)); @@ -214,6 +214,7 @@ printf("after: addr=%s, port=%u, rab_id=%u\n", addr_str.ip, addr_str.port, rab_id); encoded = ranap_rab_ass_req_encode(&message.msg.raB_AssignmentRequestIEs); + OSMO_ASSERT(encoded != NULL); OSMO_ASSERT(encoded->len == sizeof(testvec_expected_out)); OSMO_ASSERT(memcmp(encoded->data, testvec_expected_out, encoded->len) == 0); diff --git a/tests/ranap_rab_ass/ranap_rab_ass_test.ok b/tests/ranap_rab_ass/ranap_rab_ass_test.ok index 47fb97f..73565af 100644 --- a/tests/ranap_rab_ass/ranap_rab_ass_test.ok +++ b/tests/ranap_rab_ass/ranap_rab_ass_test.ok @@ -1,4 +1,3 @@ -ranap_rab_ass_req_encode ok INPUT: 0000005900000100364052000001003500487822cd80102fa7201a2c0000f44c080a028000514000272028140067400000222814003c40000000503d02000227c03500010a0901a200000000000000000000000000401f760000400100 RESULT: 0000005900000100364052000001003500487822cd80102fa7201a2c0000f44c080a028000514000272028140067400000222814003c40000000503d02000227c03500010a0901a200000000000000000000000000401f760000400100 ranap_rab_ass_resp_encode rc=46 -- To view, visit https://gerrit.osmocom.org/c/osmo-hnbgw/+/28837 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-hnbgw Gerrit-Branch: master Gerrit-Change-Id: I16fea7b2a8cb1d693e01c91d7633550e2e599ceb Gerrit-Change-Number: 28837 Gerrit-PatchSet: 3 Gerrit-Owner: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: dexter <pmaier(a)sysmocom.de> Gerrit-Reviewer: neels <nhofmeyr(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-MessageType: merged

2 years, 10 months

1
0
0 0

Change in osmo-hnbgw[master]: tests/ranap_rab_ass: fix potential NULL pointer dereferences

by neels

Attention is currently required from: fixeria, dexter. neels has posted comments on this change. ( https://gerrit.osmocom.org/c/osmo-hnbgw/+/28837 ) Change subject: tests/ranap_rab_ass: fix potential NULL pointer dereferences ...................................................................... Patch Set 3: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmo-hnbgw/+/28837 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-hnbgw Gerrit-Branch: master Gerrit-Change-Id: I16fea7b2a8cb1d693e01c91d7633550e2e599ceb Gerrit-Change-Number: 28837 Gerrit-PatchSet: 3 Gerrit-Owner: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: dexter <pmaier(a)sysmocom.de> Gerrit-Reviewer: neels <nhofmeyr(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Attention: dexter <pmaier(a)sysmocom.de> Gerrit-Comment-Date: Mon, 01 Aug 2022 14:21:25 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes Gerrit-MessageType: comment

2 years, 10 months

1
0
0 0

Change in osmo-ci[master]: scripts/obs: rewrite pushing source pkgs to OBS

by osmith

osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/28882 ) Change subject: scripts/obs: rewrite pushing source pkgs to OBS ...................................................................... scripts/obs: rewrite pushing source pkgs to OBS Harald requested that the OBS scripts should not stop if building one specific source package fails, instead it should keep going and report at the end a non-success exit code. Given that the shell script code has historically grown and became hard to maintain, I decided to rewrite the scripts for implementing this feature. This rewrite solves additional problems: * No full checkout of an OBS project like network:osmocom:latest anymore, with lots of packages that won't get updated (e.g. the uhd package has a uhd-images_3.14.1.1.tar.xz file that is 108 MB). With the old code, developers had to wait minutes during the checkout before the script reaches code that is currently being developed. Now only single packages get checked out right before they get updated. * No need to clone git repositories over and over. With the new code, git repos only get cloned if needed (for latest it is not needed if the remote git tag is the same as the version in OBS). During development, the cloned git repositories are cached. * Output from commands like "git tag -l" is not written to the log unless they failed. This makes the log more readable, which is especially important when a package fails to build, we keep going and need to spot the build error in the middle of the log later on. * No more duplicated code for nightly and latest scripts that worked similar but had slight differences. Also the list of packages is not duplicated for nightly and latest anymore; nightly uses all packages and latest uses packages that have at least one git tag. * Building source packages is decoupled from uploading them. A separate script build_srcpkg.py can be used to just build the deb + rpm spec source packages, without interacting with the OBS server. * The scripts can optionally run in docker with a command-line switch, and this is used by jenkins. This way we don't need to install more dependencies on the host such as rebar3 which is now needed for erlang/osmo_dia2gsup. * Add erlang/osmo_dia2gsup and run its generate_build_dep.sh (SYS#6006) I have done the new implementation in python to make use of argparse and to be able to use try/except and print a trace when building one package fails. Example output: * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_nightly_obs.osmocom.org… * https://jenkins.osmocom.org/jenkins/job/Osmocom_OBS_latest_obs.osmocom.org/… Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 --- M .gitignore M jobs/osmocom-obs.yml D scripts/common-obs-conflict.sh D scripts/common-obs.sh A scripts/obs/build_srcpkg.py A scripts/obs/data/Dockerfile A scripts/obs/data/Release.key A scripts/obs/data/rpmlintrc A scripts/obs/lib/__init__.py A scripts/obs/lib/config.py A scripts/obs/lib/debian.py A scripts/obs/lib/docker.py A scripts/obs/lib/git.py A scripts/obs/lib/metapkg.py A scripts/obs/lib/osc.py A scripts/obs/lib/rpm_spec.py A scripts/obs/lib/srcpkg.py A scripts/obs/update_obs_project.py D scripts/osmocom-latest-packages.sh D scripts/osmocom-next-packages.sh D scripts/osmocom-nightly-packages.sh 21 files changed, 1,338 insertions(+), 1,016 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/82/28882/1 diff --git a/.gitignore b/.gitignore index 78e48b5..1009a64 100644 --- a/.gitignore +++ b/.gitignore @@ -6,12 +6,10 @@ jenkins_jobs.ini jenkins-jobs.ini __pycache__/ +_cache/ _temp/ _deps/ _release_tarballs/ _docker_playground _repo_install_test_data/ _repo_install_test_cache/ - -# osmocom-nightly-packages.sh -nightly-3g_* diff --git a/jobs/osmocom-obs.yml b/jobs/osmocom-obs.yml index cfd3bff..ddb71f1 100644 --- a/jobs/osmocom-obs.yml +++ b/jobs/osmocom-obs.yml @@ -4,9 +4,12 @@ jobs: - Osmocom_OBS_{type}_{server} type: - - nightly - - latest - # - next (disabled: OS#5322) + - nightly: + # For nightly we don't provide ABI compatibility, make sure packages + # from different build dates are not mixed by accident + conflict_version: "$(date +%Y%m%d%H%M)" + - latest: + conflict_version: "" server: - build.opensuse.org: proj: "network:osmocom" @@ -28,10 +31,16 @@ default: 'refs/remotes/origin/master' builders: - shell: | - export PROJ={proj}:{type} - export OBS_SERVER={server} + export PYTHONUNBUFFERED=1 - ./scripts/osmocom-{type}-packages.sh + ./scripts/obs/update_obs_project.py \ + --apiurl {server} \ + --conflict-version {conflict_version} \ + --docker \ + --feed {type} \ + --git-fetch \ + --meta \ + {proj}:{type} scm: - git: branches: diff --git a/scripts/common-obs-conflict.sh b/scripts/common-obs-conflict.sh deleted file mode 100644 index 98c8b70..0000000 --- a/scripts/common-obs-conflict.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/sh -# Create conflicting dummy packages in OBS (opensuse build service), so users can't mix packages -# built from different branches by accident - -OSMO_OBS_CONFLICT_PKGVER="${OSMO_OBS_CONFLICT_PKGVER:-1.0.0}" - -# Create the conflicting package for debian -# -# $1: name of dummy package (e.g. "osmocom-nightly") -# $2-*: name of conflicting packages (e.g. "osmocom-latest") -# -# Generates the following directory structure: -# debian -# ├── changelog -# ├── compat -# ├── control -# ├── copyright -# ├── rules -# └── source -# └── format -osmo_obs_prepare_conflict_deb() { - local pkgname="$1" - shift - local oldpwd="$PWD" - - mkdir -p "debian/source" - cd "debian" - - # Fill control - cat << EOF > control -Source: ${pkgname} -Section: unknown -Priority: optional -Maintainer: Oliver Smith <osmith(a)sysmocom.de> -Build-Depends: debhelper (>= 9) -Standards-Version: 3.9.8 - -Package: ${pkgname} -Depends: \${misc:Depends} -Architecture: any -EOF - printf "Conflicts: " >> control - first=1 - for i in "$@"; do - if [ "$first" -eq 1 ]; then - first=0 - else - printf ", " >> control - fi - printf "%s" "$i" >> control - done - printf "\n" >> control - cat << EOF >> control -Description: Dummy package, which conflicts with: $@ -EOF - - # Fill changelog - cat << EOF > changelog -${pkgname} (${OSMO_OBS_CONFLICT_PKGVER}) unstable; urgency=medium - - * Dummy package, which conflicts with: $@ - - -- Oliver Smith <osmith(a)sysmocom.de> Thu, 13 Jun 2019 12:50:19 +0200 -EOF - - # Fill rules - cat << EOF > rules -#!/usr/bin/make -f -%: - dh \$@ -EOF - - # Finish up debian dir - chmod +x rules - echo "9" > compat - echo "3.0 (native)" > source/format - touch copyright - - cd "$oldpwd" -} - -# Create the conflicting package for rpm (e.g. contrib/osmocom-nightly.spec.in). The remaining -# placeholders are replaced in osmo_obs_add_rpm_spec(). -# -# $1: name of dummy package (e.g. "osmocom-nightly") -# $2-*: name of conflicting packages (e.g. "osmocom-latest") -osmo_obs_prepare_conflict_rpm() { - local pkgname="$1" - shift - local spec_in="contrib/$pkgname.spec.in" - - mkdir -p contrib - - cat << EOF > "$spec_in" -Name: $pkgname -Version: @VERSION@ -Release: 0 -Summary: Dummy package, which conflicts with: $@ -License: AGPL-3.0-or-later -Group: Hardware/Mobile -Source: @SOURCE@ -EOF - - for i in "$@"; do - echo "Conflicts: $i" >> "$spec_in" - done - - cat << EOF >> "$spec_in" -%description -Dummy package, which conflicts with: $@ -%files -EOF - -} - -# Print names of packages that the conflict package from the current feed -# (e.g. osmocom-nightly) should conflict with (e.g. osmocom-latest, -# osmocom-next, osmocom-2021q1) -osmo_obs_prepare_conflict_args() { - for i in $FEEDS_ALL; do - if [ "$i" != "$FEED" ]; then - echo "osmocom-$i" - fi - done -} - -# Create conflicting packages, based on global $FEED and $FEEDS_ALL vars -osmo_obs_prepare_conflict() { - local pkgname="osmocom-$FEED" - local conflict_args="$(osmo_obs_prepare_conflict_args)" - local oldpwd="$PWD" - - mkdir -p "$pkgname" - cd "$pkgname" - - osmo_obs_prepare_conflict_deb "$pkgname" $conflict_args - osmo_obs_prepare_conflict_rpm "$pkgname" $conflict_args - - # Put in git repository - git init . - git add -A - git commit -m "auto-commit: $pkgname dummy package" || true - git tag -f "$OSMO_OBS_CONFLICT_PKGVER" - - cd "$oldpwd" -} diff --git a/scripts/common-obs.sh b/scripts/common-obs.sh deleted file mode 100644 index 6f96e95..0000000 --- a/scripts/common-obs.sh +++ /dev/null @@ -1,326 +0,0 @@ -#!/bin/sh -# Various common code used in the OBS (opensuse build service) related osmo-ci shell scripts -. "$(dirname "$0")/common-obs-conflict.sh" - -FEEDS_ALL=" - 2021q1 - 2021q4 - 2022q1 - 2022q2 - latest - next - nightly -" - -osmo_cmd_require \ - dch \ - dh \ - dpkg-buildpackage \ - gbp \ - git \ - meson \ - mktemp \ - osc \ - patch \ - sed - -if [ -z "$PROJ" ]; then - echo "PROJ environment variable is not set" - exit 1 -fi - -if [ -z "$OBS_SERVER" ]; then - echo "OBS_SERVER environment variable is not set" - exit 1 -fi - -# Related configuration sections are in .oscrc (OS#5557) -echo "Using OBS server: $OBS_SERVER" -shopt -s expand_aliases -alias osc="osc -A '$OBS_SERVER'" - -# Add dependency to all (sub)packages in debian/control and commit the change. -# $1: path to debian/control file -# $2: package name (e.g. "libosmocore") -# $3: dependency package name (e.g. "osmocom-nightly") -# $4: dependency package version (optional, e.g. "1.0.0.202101151122") -osmo_obs_add_depend_deb() { - local d_control="$1" - local pkgname="$2" - local depend="$3" - local dependver="$4" - - if [ "$pkgname" = "$depend" ]; then - echo "NOTE: skipping dependency on itself: $depend" - return - fi - - if [ -n "$dependver" ]; then - depend="$depend (= $dependver)" - fi - - # Note: adding the comma at the end should be fine. If there is a Depends: line, it is most likely not empty. It - # should at least have ${misc:Depends} according to lintian. - sed "s/^Depends: /Depends: $depend, /g" -i "$d_control" - - git -C "$(dirname "$d_control")" commit -m "auto-commit: debian: depend on $depend" . -} - -# Add dependency to all (sub)packages in rpm spec file -# $1: path to rpm spec file -# $2: package name (e.g. "libosmocore") -# $3: dependency package name (e.g. "osmocom-nightly") -# $4: dependency package version (optional, e.g. "1.0.0.202101151122") -osmo_obs_add_depend_rpm() { - local spec="$1" - local pkgname="$2" - local depend="$3" - local dependver="$4" - - if [ "$pkgname" = "$depend" ]; then - echo "NOTE: skipping dependency on itself: $depend" - return - fi - - if [ -n "$dependver" ]; then - depend="$depend = $dependver" - fi - - ( while IFS= read -r line; do - echo "$line" - - case "$line" in - # Main package - "Name:"*) - echo "Requires: $depend" - ;; - # Subpackages - "%package"*) - echo "Requires: $depend" - ;; - # Build recipe - "%build"*) - if [ -n "$dependver" ]; then - cat << EOF -# HACK: don't let rpmlint abort the build when it finds that a library depends -# on a package with a specific version. The path used here is listed in: -# https://build.opensuse.org/package/view_file/devel:openSUSE:Factory:rpmlint… -# Instead of writing to the SOURCES dir, we could upload osmocom-rpmlintrc as -# additional source for each package. But that's way more effort, not worth it. -echo "setBadness('shlib-fixed-dependency', 0)" \\ - > "%{_sourcedir}/osmocom-rpmlintrc" - -EOF - fi - ;; - esac - done < "$spec" ) > "$spec.new" - - mv "$spec.new" "$spec" -} - -# Copy a project's rpm spec.in file to the osc package dir, set the version/source, depend on the conflicting dummy -# package and 'osc add' it -# $1: oscdir (path to checked out OSC package) -# $2: repodir (path to git repository) -# $3: package name (e.g. "libosmocore") -# $4: dependency package name (e.g. "osmocom-nightly") -# $5: dependency package version (optional, e.g. "1.0.0.202101151122") -osmo_obs_add_rpm_spec() { - local oscdir="$1" - local repodir="$2" - local name="$3" - local depend="$4" - local dependver="$5" - local spec_in="$(find "$repodir" -name "$name.spec.in")" - local spec="$oscdir/$name.spec" - local tarball - local version - local epoch - - if [ -z "$spec_in" ]; then - echo "WARNING: RPM spec missing: $name.spec.in" - return - fi - - cp "$spec_in" "$spec" - - osmo_obs_add_depend_rpm "$spec" "$name" "$depend" "$dependver" - - # Set version and epoch from "Version: [EPOCH:]VERSION" in .dsc - version="$(grep "^Version: " "$oscdir"/*.dsc | cut -d: -f2- | xargs)" - case $version in - *:*) - epoch=$(echo "$version" | cut -d : -f 1) - version=$(echo "$version" | cut -d : -f 2) - ;; - esac - if [ -n "$epoch" ]; then - sed -i "s/^Version:.*/Version: $version\nEpoch: $epoch/g" "$spec" - else - sed -i "s/^Version:.*/Version: $version/g" "$spec" - fi - - # Set source file - tarball="$(cd "$oscdir" && ls -1 *_*.tar.*)" - sed -i "s/^Source:.*/Source: $tarball/g" "$spec" - - osc add "$spec" -} - -# Get the path to a distribution specific patch, either from osmo-ci.git or from the project repository. -# $PWD must be the project repository dir. -# $1: distribution name (e.g. "debian8") -# $2: project repository (e.g. "osmo-trx", "limesuite") -osmo_obs_distro_specific_patch() { - local distro="$1" - local repo="$2" - local ret - - ret="$OSMO_CI_DIR/obs-patches/$repo/build-for-$distro.patch" - if [ -f "$ret" ]; then - echo "$ret" - return - fi - - ret="debian/patches/build-for-$distro.patch" - if [ -f "$ret" ]; then - echo "$ret" - return - fi -} - -# Check if checkout or build of a given package should be skipped, based on the -# PACKAGES environment variable. -# $1: package name (e.g. "libosmocore") -osmo_obs_skip_pkg() { - local pkgname="$1" - - if [ -z "$PACKAGES" ]; then - # Don't skip - return 1 - fi - - for i in "osmocom-$FEED" $PACKAGES; do - if [ "$i" = "$pkgname" ]; then - return 1 - fi - done - - # Skip - return 0 -} - -# Copy an already checked out repository dir and apply a distribution specific patch. -# $PWD must be where all repositories are checked out in subdirs. -# $1: distribution name (e.g. "debian8") -# $2: project repository (e.g. "osmo-trx", "limesuite") -osmo_obs_checkout_copy() { - local distro="$1" - local repo="$2" - local patch - - if osmo_obs_skip_pkg "$repo"; then - return - fi - - echo - echo "====> Checking out $repo-$distro" - - # Verify distro name for consistency - local distros=" - debian8 - debian10 - " - local found=0 - local distro_i - for distro_i in $distros; do - if [ "$distro_i" = "$distro" ]; then - found=1 - break - fi - done - if [ "$found" -eq 0 ]; then - echo "ERROR: invalid distro name: $distro, should be one of: $distros" - exit 1 - fi - - # Copy - if [ -d "$repo-$distro" ]; then - rm -rf "$repo-$distro" - fi - cp -a "$repo" "$repo-$distro" - cd "$repo-$distro" - - # Commit patch - patch="$(osmo_obs_distro_specific_patch "$distro" "$repo")" - if [ -z "$patch" ]; then - echo "ERROR: no patch found for distro=$distro, repo=$repo" - exit 1 - fi - patch -p1 < "$patch" - git commit -m "auto-commit: apply $patch" debian/ - cd .. -} - -# Run git-version-gen inside Osmocom repositories, so the .tarball-version -# becomes part of the source repository. Usually this would be done with -# "make dist", but we use git-buildpackage instead. -osmo_obs_git_version_gen() { - if [ -x ./git-version-gen ]; then - ./git-version-gen . > .tarball-version 2>/dev/null - fi -} - -# Return a version based on the latest tag and commit (e.g. "1.5.1.93.47cc") -# or fall back to the last debian version (e.g. "2.2.6"). Run -# osmo_obs_git_version_gen before. $PWD must be inside a git repository. -osmo_obs_get_commit_version() { - local version="" - - if [ -e ".tarball-version" ]; then - version=$(cat .tarball-version) - # debian doesn't allow '-' in version. - version=$(echo "$version" | sed 's/-/./g' ) - fi - - # deb version - deb_version=$(head -1 debian/changelog | cut -d ' ' -f 2 | sed 's,(,,' | sed 's,),,') - if [ -z "$version" ] || [ "$version" = "UNKNOWN" ]; then - version="$deb_version" - else - # add epoch from debian/changelog - case $deb_version in - *:*) - epoch=$(echo "$deb_version" | cut -d : -f 1) - version=$epoch:$version - ;; - esac - fi - - echo -n "$version" -} - -# Verify that $FEED is in $FEEDS and $FEEDS_ALL -osmo_obs_verify_feed() { - local i - local j - - for i in $FEEDS; do - if [ "$i" != "$FEED" ]; then - continue - fi - - for j in $FEEDS_ALL; do - if [ "$j" = "$i" ]; then - return - fi - done - - echo "feed found in FEEDS but not FEEDS_ALL: $FEED" - exit 1 - done - - echo "unsupported feed: $FEED" - exit 1 -} diff --git a/scripts/obs/build_srcpkg.py b/scripts/obs/build_srcpkg.py new file mode 100755 index 0000000..d41e78b --- /dev/null +++ b/scripts/obs/build_srcpkg.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import argparse +import lib +import lib.config +import lib.docker +import lib.git +import lib.metapkg +import lib.srcpkg + + +def main(): + parser = argparse.ArgumentParser( + description="For one or more packages, clone the git repository and" + " build the debian source package as well as an rpm .spec" + " file. This is the same code that runs to generate source" + " packages which we upload to https://obs.osmocom.org." + f" Output dir: {lib.config.path_temp}/srcpkgs") + lib.add_shared_arguments(parser) + parser.add_argument("package", nargs="?", + help="package name, e.g. libosmocore or open5gs") + args = parser.parse_args() + + if not args.meta and not args.package: + print("ERROR: specify -m and/or a package. See -h for help.") + exit(1) + + lib.set_cmds_verbose(args.verbose) + + if args.docker: + lib.docker.run_in_docker_and_exit(__file__, args) + + lib.check_required_programs() + if args.package: + lib.check_package(args.package) + lib.remove_temp() + + if args.meta: + lib.metapkg.build(args.feed, args.conflict_version) + + if args.package: + lib.srcpkg.build(args.package, args.feed, args.conflict_version, + args.git_fetch) + + +if __name__ == "__main__": + main() diff --git a/scripts/obs/data/Dockerfile b/scripts/obs/data/Dockerfile new file mode 100644 index 0000000..d8bd9c5 --- /dev/null +++ b/scripts/obs/data/Dockerfile @@ -0,0 +1,35 @@ +FROM debian:bullseye +ARG UID + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + gnupg2 \ + && \ + apt-get clean + +COPY Release.key /tmp/Release.key +RUN apt-key add /tmp/Release.key && \ + rm /tmp/Release.key && \ + echo "deb https://downloads.osmocom.org/packages/osmocom:/latest/Debian_11/ ./" \ + > /etc/apt/sources.list.d/osmocom-latest.list + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + debhelper \ + dh-python \ + dpkg-dev \ + fakeroot \ + git \ + meson \ + osc \ + python3-setuptools \ + rebar3 \ + sed \ + && \ + apt-get clean + +RUN useradd --uid=${UID} -m user +USER user diff --git a/scripts/obs/data/Release.key b/scripts/obs/data/Release.key new file mode 100644 index 0000000..ecca084 --- /dev/null +++ b/scripts/obs/data/Release.key @@ -0,0 +1,26 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.5 (GNU/Linux) + +mQENBGKzE1QBCADFcM3ZzggvgxNRNNqDGWf5xIDAiK5qzFLdGes7L6F9VCHdaPy0 +RAOB5bFb/Q1tSDFNEBLtaauXKz+4iGL6qMVjZcyjzpB5w4jKN+kkrFRhjDNUv/SH +BX6d+P7v5WBGSNArNgA8D1BGzckp5a99EZ0okMJFEqIcN40PD6OGugpq5XnVV5Nk +e93fLa2Cu8vhFBcVn6CuHeEhsmuMf6NLbQRNfNNCEEUYaZn7beMYtpZ7t1djsKx5 +1xGm50OzI22FLu8lELQ9d7qMVGRG3WHYawX9BDteRybiyqxfwUHm1haWazRJtlGt +UWyzvwAb80BK1J2Nu5fbAa3w5CoEPAbUuCyrABEBAAG0JW9zbW9jb20gT0JTIFBy +b2plY3QgPG9zbW9jb21Ab3Ntb2NvbT6JAVQEEwEIAD4WIQRrKp83ktFetw1Oao+G +pzC2U3JZcwUCYrMV4wIbAwUJBB6yjwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAK +CRCGpzC2U3JZc4FRCACQQkKIrnvQ7n2u7GSmyVZa3I+oLoFXSGqaGyey5TW/nrMm +vFDKU3qliHiuNSmUY35SnAhXUsvqOYppxVRoO1MLrqUvzMOnIWqkJpf8mtjGUnsW +jyVeto7Rsjs75y2i1Hk+e7ljb/V65J3NlfrfEYWbqR9AKd53ReNXTdrQ0J05A38N +GdI4Ld/2lNISAwaBmGhqdeKsLHpQw/JERU1TApVJR1whFiIwDF1rOCg9GPnNKIk7 +yRZdK267XzztrainX/cbPILyzUZEDhYs6wQuyACyQ1YUxZIxrwVfk7PMNay8CrLH +z42B73Ne5IAj8+op/3iJafFONLm7YXiDUFN+QDYAiQEzBBMBCAAdFiEExoiYhHND +S7aVYlnqa51NyAUyjdsFAmKzE1UACgkQa51NyAUyjdvuZgf+OXmr//i7u7Gg7eWB +7e0qUsyCId9lXS8J437x3K6ciJfD7/6RSy8TFW5Nglm/uSkbyq582I8t+SoOirMD +E6cg9U/5+h5s46bAf+Kd2XS/6tLGeNLM18i4el8CP06NpFzDrsKu76uYFpyRiiHD +otBdtgxeLJ83LugGfZslF+/5cigJkAJMhAdVvGO8h85R6fba8ZSOKtMKkaQRfi76 +nhyOrJPlLuS+DLEnHwdkOFgtKnxHdjM97K+Tx0gisb6uwaWroXfSLnhP8RTLLZZy +Z+noU1Hw3c+mn4c/NYbcC/uwHYHKRzuf9gHnQ3dGgv0Z5sbeLRVo92hjGj7Ftlyd +4hmKBg== +=HxK4 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/obs/data/rpmlintrc b/scripts/obs/data/rpmlintrc new file mode 100644 index 0000000..aa3903c --- /dev/null +++ b/scripts/obs/data/rpmlintrc @@ -0,0 +1,5 @@ +# Don't abort the build when finding a library that depends on a package with +# a specific version. This is intentional for nightly builds, we don't want +# libraries from different build dates to be mixed as they might have ABI +# incompatibilities. +setBadness('shlib-fixed-dependency', 0) diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py new file mode 100644 index 0000000..193c248 --- /dev/null +++ b/scripts/obs/lib/__init__.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import importlib +import os +import shutil +import subprocess +import sys +import tempfile +import lib.config + +cmds_verbose = False + + +def add_shared_arguments(parser): + parser.add_argument("-f", "--feed", help="package feed (default: nightly)", + metavar="FEED", default="nightly", + choices=lib.config.feeds) + parser.add_argument("-d", "--docker", + help="run in docker to avoid installing required pkgs", + action="store_true") + parser.add_argument("-g", "--git-fetch", + help="fetch already cloned git repositories", + action="store_true") + parser.add_argument("-m", "--meta", action="store_true", + help="build a meta package (e.g. osmocom-nightly)") + parser.add_argument("-c", "--conflict-version", nargs="?", + help="Of the generated source packages, all Osmocom" + " packages (not e.g. open5gs, see lib/config.py" + " for full list) depend on a meta-package such as" + " osmocom-nightly, osmocom-latest, osmocom-2021q1" + " etc. These meta-packages conflict with each" + " other to ensure that one does not mix e.g." + " latest and nightly packages by accident." + " With this -c argument, it is possible to let" + " these packages depend on a meta-package of a" + " specific version. This is used for nightly and" + " 20YYqX packages to ensure they are not mixed" + " from different build dates (ABI compatibility" + " is only on latest).") + parser.add_argument("-v", "--verbose", action="store_true", + help="always print shell commands and their output," + " instead of only printing them on error") + + +def set_cmds_verbose(new_val): + global cmds_verbose + cmds_verbose = new_val + + +def check_required_programs(): + ok = True + + for program in lib.config.required_programs: + if not shutil.which(program): + print(f"ERROR: missing program: {program}") + ok = False + + for module in lib.config.required_python_modules: + if not importlib.find_loader(module): + print(f"ERROR: missing python3 module: {module}") + ok = False + + if not ok: + print("Either install them or use the -d argument to run in docker") + exit(1) + + +def check_package(package): + if package in lib.config.projects_osmocom: + return + if package in lib.config.projects_other: + return + + print(f"ERROR: unknown package: {package}") + print("See packages_osmocom and packages_other in obs/lib/config.py") + exit(1) + + +def exit_error_cmd(completed, error_msg): + """ :param completed: return from run_cmd() below """ + print() + print(f"ERROR: {error_msg}") + print() + print(f"*** command ***\n{completed.args}\n") + print(f"*** returncode ***\n{completed.returncode}\n") + print(f"*** output ***\n{completed.output}") + print("*** python trace ***") + raise RuntimeError("shell command related error, find details right above" + " this python trace") + + +def run_cmd(cmd, check=True, *args, **kwargs): + """ Like subprocess.run, but has check=True and text=True by default and + allows capturing the output while displaying it at the same time. By + default the output is hidden unless there's an error, with -v the + output gets written to stdout. + :returns: subprocess.CompletedProcess instance, but with combined + stdout + stderr written to ret.output + :param check: stop with error if exit code is not 0 """ + global cmds_verbose + + if cmds_verbose: + print(f"+ {cmd}") + + with tempfile.TemporaryFile(encoding="utf8", mode="w+") as output_buf: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, bufsize=1, + *args, **kwargs) + + while True: + out = p.stdout.read(1) + if out == "" and p.poll() is not None: + break + if out != "": + output_buf.write(out) + if cmds_verbose: + sys.stdout.write(out) + sys.stdout.flush() + + output_buf.seek(0) + setattr(p, "output", output_buf.read()) + + if p.returncode == 0 or not check: + return p + + exit_error_cmd(p, "command failed unexpectedly") + + +def remove_temp(): + run_cmd(["rm", "-rf", lib.config.path_temp]) + + +def remove_cache_extra_files(): + """ dpkg-buildpackage outputs all files to the top dir of the package + dir, so it will always put them in _cache when building e.g. the debian + source package of _cache/libosmocore. Clear all extra files from _cache + that don't belog to the git repositories which we actually want to + cache. """ + run_cmd(["find", lib.config.path_cache, "-maxdepth", "1", "-type", "f", + "-delete"]) + + +def get_output_path(project): + return f"{lib.config.path_temp}/srcpkgs/{os.path.basename(project)}" diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py new file mode 100644 index 0000000..22c9c05 --- /dev/null +++ b/scripts/obs/lib/config.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os + +# Lists are ordered alphabetically. + +path_top = os.path.normpath(f"{os.path.realpath(__file__)}/../..") +path_cache = f"{path_top}/_cache" +path_temp = f"{path_top}/_temp" + +# Keep in sync with packages installed in data/Dockerfile +required_programs = [ + "dh", + "dh_python3", + "dpkg-buildpackage", + "fakeroot", + "find", + "git", + "meson", + "osc", + "rebar3", + "sed", +] + +required_python_modules = [ + "setuptools", +] + +feeds = [ + "2022q1", + "2022q2", + "latest", + "nightly", +] + +# Osmocom projects: generated source packages will depend on a meta package, +# such as osmocom-nightly, osmocom-latest or osmocom-2022q1. This meta package +# prevents that packages from different feeds are mixed by accident. +projects_osmocom = [ + "erlang/osmo_dia2gsup", + "libasn1c", + "libgtpnl", + "libosmo-abis", + "libosmo-dsp", + "libosmo-netif", + "libosmo-sccp", + "libosmocore", + "libsmpp34", + "libusrp", + "osmo-bsc", + "osmo-bts", + "osmo-cbc", + "osmo-e1d", + "osmo-gbproxy", + "osmo-ggsn", + "osmo-gsm-manuals", + "osmo-hlr", + "osmo-hnbgw", + "osmo-hnodeb", + "osmo-iuh", + "osmo-mgw", + "osmo-msc", + "osmo-pcap", + "osmo-pcu", + "osmo-remsim", + "osmo-sgsn", + "osmo-sip-connector", + "osmo-smlc", + "osmo-sysmon", + "osmo-trx", + "osmo-uecups", + "python/osmo-python-tests", + "rtl-sdr", + "simtrace2", +] +projects_other = [ + "limesuite", + "neocon", + "open5gs", +] + +git_url_default = "https://gerrit.osmocom.org" # /project gets appended +git_url_other = { + "libosmo-dsp": "https://gitea.osmocom.org/sdr/libosmo-dsp", + "limesuite": "https://github.com/myriadrf/LimeSuite", + "neocon": "https://github.com/laf0rge/neocon", + "open5gs": "https://github.com/open5gs/open5gs", + "rtl-sdr": "https://gitea.osmocom.org/sdr/rtl-sdr", +} + +git_branch_default = "master" +git_branch_other = { + "open5gs": "main", +} + +git_latest_tag_pattern_default = "^[0-9]*\\.[0-9]*\\.[0-9]*$" +git_latest_tag_pattern_other = { + "limesuite": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", + "open5gs": "^v[0-9]*\\.[0-9]*\\.[0-9]*$", +} + +docker_image_name = "debian-bullseye-osmocom-obs" diff --git a/scripts/obs/lib/debian.py b/scripts/obs/lib/debian.py new file mode 100644 index 0000000..af152da --- /dev/null +++ b/scripts/obs/lib/debian.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import datetime +import os +import shlex +import lib +import lib.git + + +def control_add_depend(project, pkgname, version): + """ :param pkgname: of the meta-package to depend on (e.g. osmocom-nightly) + :param version: of the meta-pkgname to depend on or None """ + repo_path = lib.git.get_repo_path(project) + + if version: + depend = f"{pkgname} (= {version})" + else: + depend = pkgname + + cmd = ["sed", f"s/^Depends: /Depends: {depend}, /", "-i", "debian/control"] + lib.run_cmd(cmd, cwd=repo_path) + + +def changelog_add_entry(project, feed, version): + """ :param version: for the new changelog entry """ + repo_path = lib.git.get_repo_path(project) + changelog_path = f"{repo_path}/debian/changelog" + + changelog_old = open(changelog_path).read() + + # Package name might be different from project name, read it from changelog + pkgname = changelog_old.split(" ", 1)[0] + assert pkgname + + # Debian doesn't allow '-' in version + version = version.replace("-", ".") + + # Debian changelog requires this specific date format + date = datetime.datetime.now(datetime.timezone.utc) + date_str = date.strftime("%a, %d %b %Y %H:%M:%S %z") + + # Add new changelog entry + with open(changelog_path, "w") as f: + f.write(f"{pkgname} ({version}) unstable; urgency=medium\n") + f.write("\n") + f.write(" * Automatically generated changelog entry for building the" + f" Osmocom {feed} feed\n") + f.write("\n") + f.write(f" -- Osmocom OBS scripts <info(a)osmocom.org> {date_str}\n") + f.write("\n") + f.write(changelog_old) + + +def fix_source_format(project): + """ Always use format "3.0 (native)" (e.g. limesuite has "3.0 (quilt)").""" + repo_path = lib.git.get_repo_path(project) + format_path = f"{repo_path}/debian/source/format" + if not os.path.exists(format_path): + return + + expected = "3.0 (native)\n" + current = open(format_path, "r").read() + + if current == expected: + return + + print(f"{project}: fixing debian/source/format ({current.rstrip()} =>" + f" {expected.rstrip()})") + open(format_path, "w").write(expected) + + +def get_last_version_from_changelog(project): + repo_path = lib.git.get_repo_path(project) + changelog_path = f"{repo_path}/debian/changelog" + + if not os.path.exists(changelog_path): + return None + + changelog = open(changelog_path).read() + if not changelog: + return None + + return changelog.split("(", 1)[1].split(")", 1)[0] + + +def changelog_add_entry_if_needed(project, feed, version): + """ Adjust the changelog if the version in the changelog is different from + the given version. """ + version_changelog = get_last_version_from_changelog(project) + if version_changelog == version: + return + + print(f"{project}: adding debian/changelog entry ({version_changelog} =>" + f" {version})") + changelog_add_entry(project, feed, version) + + +def build_source_package(project): + fix_source_format(project) + print(f"{project}: building debian source package") + lib.run_cmd(["dpkg-buildpackage", "-S", "-uc", "-us", "-d"], + cwd=lib.git.get_repo_path(project)) + + +def move_files_to_output(project): + path_output = lib.get_output_path(project) + lib.run_cmd(f"mv *.tar* *.dsc {shlex.quote(path_output)}", shell=True, + cwd=lib.config.path_cache) diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py new file mode 100644 index 0000000..a5dbcd9 --- /dev/null +++ b/scripts/obs/lib/docker.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import shutil +import subprocess +import sys +import lib +import lib.config + + +def build_image(): + print(f"docker: building image {lib.config.docker_image_name}") + lib.run_cmd(["docker", "build", + "--build-arg", f"UID={os.getuid()}", + "-t", lib.config.docker_image_name, + f"{lib.config.path_top}/data"]) + + +def get_oscrc(): + ret = os.path.expanduser("~/.oscrc") + if "OSCRC" in os.environ: + ret = os.environ["OSCRC"] + + if os.path.exists(ret): + return os.path.realpath(ret) + + print("ERROR: couldn't find ~/.oscrc. Put it there or set OSCRC.") + exit(1) + + +def run_in_docker_and_exit(script_path, args, add_oscrc=False): + if "INSIDE_DOCKER" in os.environ: + return + + if not shutil.which("docker"): + print("ERROR: docker is not installed") + exit(1) + + oscrc = None + if add_oscrc: + oscrc = get_oscrc() + + # Build the docker image. Unless it is up-to-date, this will take a few + # minutes or so, therefore print the output. + lib.set_cmds_verbose(True) + build_image() + lib.set_cmds_verbose(args.verbose) + + cmd = ["docker", "run", + "-e", "INSIDE_DOCKER=1", + "-e", "PYTHONUNBUFFERED=1", + "--rm", "-v", f"{lib.config.path_top}:/obs"] + + if oscrc: + cmd += ["-v", f"{oscrc}:/home/user/.oscrc"] + + script_path = f"/obs/{os.path.basename(script_path)}" + cmd += [lib.config.docker_image_name, script_path] + sys.argv[1:] + + print(f"docker: running: {os.path.basename(script_path)} inside docker") + ret = subprocess.run(cmd) + exit(ret.returncode) diff --git a/scripts/obs/lib/git.py b/scripts/obs/lib/git.py new file mode 100644 index 0000000..67f1d5e --- /dev/null +++ b/scripts/obs/lib/git.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import re +import lib.config + + +def get_repo_path(project): + return f"{lib.config.path_cache}/{os.path.basename(project)}" + + +def get_repo_url(project): + if project in lib.config.git_url_other: + return lib.config.git_url_other[project] + return f"{lib.config.git_url_default}/{project}" + + +def get_latest_tag_pattern(project): + if project in lib.config.git_latest_tag_pattern_other: + return lib.config.git_latest_tag_pattern_other[project] + return lib.config.git_latest_tag_pattern_default + + +def clone(project, fetch=False): + repo_path = get_repo_path(project) + url = get_repo_url(project) + + if os.path.exists(repo_path): + if fetch: + print(f"{project}: cloning {url} (cached, fetching)") + lib.run_cmd(["git", "fetch"], cwd=repo_path) + else: + print(f"{project}: cloning {url} (cached, not fetching)") + return + + print(f"{project}: cloning {url}") + os.makedirs(lib.config.path_cache, exist_ok=True) + lib.run_cmd(["git", "clone", url, repo_path]) + + lib.run_cmd(["git", "config", "user.name", "Osmocom OBS scripts"], + cwd=repo_path) + lib.run_cmd(["git", "config", "user.email", "info(a)osmocom.org"], + cwd=repo_path) + + +def clean(project): + repo_path = get_repo_path(project) + lib.run_cmd(["git", "clean", "-ffxd"], cwd=repo_path) + + +def checkout(project, branch): + repo_path = get_repo_path(project) + print(f"{project}: checking out {branch}") + lib.run_cmd(["git", "checkout", "-f", branch], cwd=repo_path) + lib.run_cmd(["git", "reset", "--hard", branch], cwd=repo_path) + + +def checkout_default_branch(project): + branch = lib.config.git_branch_default + if project in lib.config.git_branch_other: + branch = lib.config.git_branch_other[project] + + checkout(project, f"origin/{branch}") + + +def get_latest_tag(project): + pattern_str = get_latest_tag_pattern(project) + pattern = re.compile(pattern_str) + repo_path = get_repo_path(project) + + git_tag_ret = lib.run_cmd(["git", "tag", "-l", "--sort=-v:refname"], + cwd=repo_path) + + for line in git_tag_ret.output.split('\n'): + line = line.strip('\r') + if pattern.match(line): + return line + + lib.exit_error_cmd(git_tag_ret, f"couldn't find latest tag for {project}," + f" regex used on output: {pattern_str}") + + +def get_latest_tag_remote(project): + pattern_str = get_latest_tag_pattern(project) + pattern = re.compile(pattern_str) + + print(f"{project}: getting latest tag from git remote") + ls_remote = lib.run_cmd(["git", "ls-remote", "--tags", "--sort=-v:refname", + get_repo_url(project)]) + for line in ls_remote.output.split('\n'): + # Tags are listed twice, skip the ones with ^{} at the end + if "^{}" in line: + continue + + if "refs/tags/" not in line: + continue + + line = line.rstrip().split("refs/tags/")[1] + if pattern.match(line): + return line + + # No tag found probably means the repository was just created and doesn't + # have a release tag yet + return None + + +def checkout_latest_tag(project): + checkout(project, get_latest_tag(project)) diff --git a/scripts/obs/lib/metapkg.py b/scripts/obs/lib/metapkg.py new file mode 100644 index 0000000..dd261de --- /dev/null +++ b/scripts/obs/lib/metapkg.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import lib +import lib.config +import lib.debian +import lib.rpm_spec + + +def get_conflicts(feed): + ret = [] + for f in lib.config.feeds: + if f == feed: + continue + ret += [f"osmocom-{f}"] + return ret + + +def prepare_source_dir(feed): + path = f"{lib.config.path_cache}/osmocom-{feed}" + + if os.path.exists(path): + lib.run_cmd(["rm", "-rf", path]) + + os.makedirs(f"{path}/debian") + os.makedirs(f"{path}/contrib") + + +def generate_debian_pkg(feed, version): + path = f"{lib.config.path_cache}/osmocom-{feed}" + conflicts = get_conflicts(feed) + + with open(f"{path}/debian/control", "w") as f: + f.write(f"Source: osmocom-{feed}\n") + f.write("Section: unknown\n") + f.write("Priority: optional\n") + f.write("Maintainer: Osmocom OBS scripts <info(a)osmocom.org>\n") + f.write("Build-Depends: debhelper (>= 10)\n") + f.write("Standards-Version: 3.9.8\n") + f.write("\n") + f.write(f"Package: osmocom-{feed}\n") + f.write("Depends: ${misc:Depends}\n") + f.write("Architecture: any\n") + f.write(f"Conflicts: {', '.join(conflicts)}\n") + f.write(f"Description: Dummy package, conflicts with {conflicts}\n") + + with open(f"{path}/debian/changelog", "w") as f: + f.write(f"osmocom-{feed} ({version}) unstable; urgency=medium\n") + f.write("\n") + f.write(f" * Dummy package, which conflicts with: {conflicts}\n") + f.write("\n") + f.write(" -- Osmocom OBS scripts <info(a)osmocom.org> Tue, 25 Jul 2022" + " 15:48:00 +0200\n") + + with open(f"{path}/debian/rules", "w") as f: + f.write("#!/usr/bin/make -f\n") + f.write("%:\n") + f.write("\tdh $@\n") + + lib.run_cmd(["chmod", "+x", f"{path}/debian/rules"]) + + with open(f"{path}/debian/compat", "w") as f: + f.write("10\n") + + +def generate_rpm_spec(feed, version): + print(f"osmocom-{feed}: generating rpm spec file") + path = (f"{lib.config.path_cache}/osmocom-{feed}/contrib/osmocom-{feed}" + ".spec.in") + conflicts = get_conflicts(feed) + + with open(path, "w") as f: + f.write(f"Name: osmocom-{feed}\n") + f.write(f"Version: {version}\n") + f.write(f"Summary: Dummy package, conflicts with: {conflicts}\n") + f.write("Release: 0\n") + f.write("License: AGPL-3.0-or-later\n") + f.write("Group: Hardware/Mobile\n") + for conflict in conflicts: + f.write(f"Conflicts: {conflict}\n") + f.write("%description\n") + f.write(f"Dummy package, which conflicts with: {conflicts}\n") + f.write("%files\n") + + +def build(feed, conflict_version): + pkgname = f"osmocom-{feed}" + version = conflict_version if conflict_version else "1.0.0" + print(f"{pkgname}: generating meta package {version}") + + prepare_source_dir(feed) + generate_debian_pkg(feed, version) + + os.makedirs(lib.get_output_path(pkgname)) + lib.remove_cache_extra_files() + + lib.debian.build_source_package(pkgname) + lib.debian.move_files_to_output(pkgname) + + generate_rpm_spec(feed, version) + lib.rpm_spec.copy_to_output(pkgname) + + lib.remove_cache_extra_files() + return version diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py new file mode 100644 index 0000000..2fcdbc6 --- /dev/null +++ b/scripts/obs/lib/osc.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> + +import glob +import os +import shlex +import shutil +import lib +import lib.config + +apiurl = None + + +def check_proj(obs_project): + if ":" in obs_project: + return + + print(f"ERROR: this doesn't look like a valid OBS project: {obs_project}") + exit(1) + + +def set_apiurl(url): + global apiurl + apiurl = url + + +def run_osc(cmd, *args, **kwargs): + global apiurl + + # For some osc commands like 'osc add *' it makes sense to use a flat + # string and shell=True, hence support both list and string in this wrapper + if isinstance(cmd, str): + if apiurl: + cmd = f"osc -A {shlex.quote(apiurl)} {cmd}" + else: + cmd = f"osc {cmd}" + else: + if apiurl: + cmd = ["osc", "-A", apiurl] + cmd + else: + cmd = ["osc"] + cmd + + return lib.run_cmd(cmd, *args, **kwargs) + + +def get_remote_pkgs(proj): + print(f"OBS: getting packages in {proj}") + ret = run_osc(["list", proj]) + return ret.output.rstrip().split("\n") + + +def get_package_version(proj, package): + print(f"{package}: getting OBS version") + ret = run_osc(["list", proj, os.path.basename(package)]) + + # Empty OBS package + if ret.output == '\n': + return "0" + + # Extract the version from the dsc filename + for line in ret.output.split('\n'): + line = line.rstrip() + + if line.endswith(".dsc"): + return line.split("_")[-1][:-4] + + lib.exit_error_cmd(ret, "failed to find package version on OBS by" + " extracting the version from the .dsc filename") + + +def create_package(proj, package): + print(f"{package}: creating new OBS package") + + # cut off repository prefix like in "python/osmo-python-tests" + package = os.path.basename(package) + + path_meta = f"{lib.config.path_temp}/_meta" + path_meta_obs = f"source/{proj}/{package}/_meta" + + with open(path_meta, "w") as f: + f.write(f'<package name="{package}" project="{proj}">\n') + f.write(f'<title>{package}</title>\n') + f.write('<description></description>\n') + f.write('</package>\n') + + run_osc(["api", "-X", "PUT", "-T", path_meta, path_meta_obs]) + + os.unlink(path_meta) + + +def remove_temp_osc(): + lib.run_cmd(["rm", "-rf", f"{lib.config.path_temp}/osc"]) + + +def update_package(proj, package, version): + print(f"{package}: updating OBS package") + + # cut off repository prefix like in "python/osmo-python-tests" + package = os.path.basename(package) + + path_output = lib.get_output_path(package) + path_temp_osc = f"{lib.config.path_temp}/osc" + path_temp_osc_pkg = f"{path_temp_osc}/{proj}/{package}" + + remove_temp_osc() + os.makedirs(path_temp_osc) + + run_osc(["checkout", proj, package], cwd=path_temp_osc) + + if glob.glob(f"{path_temp_osc_pkg}/*"): + run_osc("del *", shell=True, cwd=path_temp_osc_pkg) + + lib.run_cmd(f"mv * {shlex.quote(path_temp_osc_pkg)}", shell=True, + cwd=path_output) + + run_osc("add *", shell=True, cwd=path_temp_osc_pkg) + run_osc(["commit", "-m", f"upgrade to {version}"], cwd=path_temp_osc_pkg) + + remove_temp_osc() diff --git a/scripts/obs/lib/rpm_spec.py b/scripts/obs/lib/rpm_spec.py new file mode 100644 index 0000000..a7c4379 --- /dev/null +++ b/scripts/obs/lib/rpm_spec.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import glob +import os +import shutil +import lib +import lib.git + + +def get_spec_in_path(project): + repo_path = lib.git.get_repo_path(project) + ret = glob.glob(f"{repo_path}/**/contrib/*.spec.in", recursive=True) + assert len(ret) < 2 + return ret[0] if ret else None + + +def get_source_name(project): + """ Get the name to the .tar.xz file generated by dpkg-buildpackage, that + we also use in the rpm spec file. This requires + lib.debian.move_files_to_output() to run first.""" + path_output = lib.get_output_path(project) + ret = glob.glob(f"{path_output}/*.tar.*") + assert len(ret) == 1 + return os.path.basename(ret[0]) + + +def add_depend(project, pkgname, version): + """ Add a 'Requires: ...' line below the 'Name:' and '%package' lines of + the .spec.in file. + :param pkgname: of the meta-package to depend on (e.g. osmocom-nightly) + :param version: of the meta-pkgname to depend on or None """ + path = get_spec_in_path(project) + + req_line = f"Requires: {pkgname}" + if version: + req_line += f" = {version}" + + # Main package + lib.run_cmd(["sed", "-e", "/^Name:/a\\", "-e", req_line, "-i", path]) + + # Subpackages + lib.run_cmd(["sed", "-e", "/^%package/a\\", "-e", req_line, "-i", path]) + + +def generate(project, version, epoch): + """ Update the version and source in the .spec.in file. + :param version: from get-version-gen script """ + print(f"{project}: generating rpm spec file") + path = get_spec_in_path(project) + source = get_source_name(project) + + # rpmbuild doesn't allow '-' in version + version = version.replace("-", ".") + + # Version + lib.run_cmd(["sed", f"s/^Version:.*/Version: {version}/", "-i", path]) + if epoch: + epoch_line = f"Epoch: {epoch}" + lib.run_cmd(["sed", "-e", "/^Version:/a\\", "-e", epoch_line, "-i", + path]) + + # Source: add the one generated by dpkg-buildpackage and another line with + # rpmlintrc + lib.run_cmd(["sed", f"s/^Source:.*/Source: {source}/", "-i", path]) + rpmlint_line = "Source1: rpmlintrc" + lib.run_cmd(["sed", "-e", "/^Source:/a\\", "-e", rpmlint_line, "-i", path]) + + # rpmbuild assumes the directory inside the source tarball has the version + # at the end, but dpkg-buildpackage doesn't generate it that way. Set the + # directory with "%setup -n dirname". + lib.run_cmd(["sed", f"s/^%setup/%setup -n {os.path.basename(project)}/", + "-i", path]) + + +def copy_to_output(project): + # rpm spec + path_spec_in = get_spec_in_path(project) + path_output = lib.get_output_path(project) + name_spec = os.path.basename(path_spec_in)[:-3] # remove '.in' + shutil.copy(path_spec_in, f"{path_output}/{name_spec}") + + # rpmlintrc + shutil.copy(f"{lib.config.path_top}/data/rpmlintrc", path_output) diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py new file mode 100644 index 0000000..468a240 --- /dev/null +++ b/scripts/obs/lib/srcpkg.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import lib.config +import lib.debian +import lib.rpm_spec + + +def checkout_for_feed(project, feed): + """ checkout a commit, either latest tag or master or 20YY branch """ + if feed == "latest": + lib.git.checkout_latest_tag(project) + elif feed == "nightly": + lib.git.checkout_default_branch(project) + else: # 2022q1 etc + lib.git.checkout(project, feed) + + +def get_git_version_gen_path(project): + # Use git-version-gen in the project's repository if available + repo_path = lib.git.get_repo_path(project) + ret = f"{repo_path}/git-version-gen" + if os.path.exists(ret): + return ret + + # Use git-version-gen script from libosmocore.git as fallback + print(f"{project}: has no git-version-gen, using the one from libosmocore") + repo_path = lib.git.get_repo_path("libosmocore") + ret = f"{repo_path}/git-version-gen" + if not os.path.exists(ret): + lib.git.clone("libosmocore") + if os.path.exists(ret): + return ret + + print(f"ERROR: {project}.git doesn't have a git-version-gen script and" + " couldn't find libosmocore.git's copy of the script here either: " + + ret) + exit(1) + + +def get_git_version(project): + """ :returns: the string from git-version-gen, e.g. '1.7.0.10-76bdb' """ + repo_path = lib.git.get_repo_path(project) + script_path = get_git_version_gen_path(project) + + ret = lib.run_cmd([script_path, "."], cwd=repo_path) + if not ret.output: + lib.exit_error_cmd(ret, "empty output from git-version-gen") + + return ret.output + + +def get_version_for_feed(project, feed, conflict_version): + if feed == "latest": + # There's always a tag if we are here. If there was none, the build + # would have been skipped for latest. + ret = lib.git.get_latest_tag(project) + return ret[1:] if ret.startswith("v") else ret + + ret = get_git_version(project) + + # Try to get the last version from the debian/changelog if we can't get + # it with git-version-gen, like it was done in the previous OBS scripts + if ret == "UNKNOWN": + ret = lib.debian.get_last_version_from_changelog(project) + # cut off epoch, we retrieve it separately in get_epoch() below + if ":" in ret: + ret = ret.split(":")[1] + + if not ret: + ret = "0.0.0" + + # Append the conflict_version to increase the version even if the commit + # did not change (OS#5135) + if conflict_version: + ret = f"{ret}.{conflict_version}" + + return ret + + +def get_epoch(project): + """ The osmo-gbproxy used to have the same package version as osmo-sgsn + until 2021 where it was split into its own git repository. From then + on, osmo-gbproxy has a 0.*.* package version, which is smaller than + the previous 1.*.* from osmo-sgsn. We had to set the epoch to 1 for + osmo-gbproxy so package managers know these 0.*.* versions are higher + than the previous 1.*.* ones that are still found in e.g. debian 11. + The epoch is set in debian/changelog, retrieve it from there. + :returns: the epoch number if set, e.g. "1" or an empty string """ + version_epoch = lib.debian.get_last_version_from_changelog(project) + + if ":" in version_epoch: + return version_epoch.split(":")[0] + + return "" + + +def prepare_project_osmo_dia2gsup(): + """ Run erlang/osmo_dia2gsup's generate_build_dep.sh script to download + sources for dependencies. """ + lib.run_cmd("contrib/generate_build_dep.sh", + cwd=lib.git.get_repo_path("erlang/osmo_dia2gsup")) + + +def prepare_project_open5gs(): + """ Build fails without downloading freeDiameter sources. Also we can't + just update all subprojects because it would fail with 'Subproject + exists but has no meson.build file' for promethous-client-c. """ + lib.run_cmd(["meson", "subprojects", "download", "freeDiameter"], + cwd=lib.git.get_repo_path("open5gs")) + + +def write_tarball_version(project, version): + repo_path = lib.git.get_repo_path(project) + + with open(f"{repo_path}/.tarball-version", "w") as f: + f.write(f"{version}\n") + + +def build(project, feed, conflict_version, fetch): + lib.git.clone(project, fetch) + lib.git.clean(project) + checkout_for_feed(project, feed) + version = get_version_for_feed(project, feed, conflict_version) + epoch = get_epoch(project) + version_epoch = f"{epoch}:{version}" if epoch else version + has_rpm_spec = lib.rpm_spec.get_spec_in_path(project) is not None + + print(f"{project}: building source package {version_epoch}") + write_tarball_version(project, version_epoch) + + if project in lib.config.projects_osmocom: + metapkg = f"osmocom-{feed}" + lib.debian.control_add_depend(project, metapkg, conflict_version) + if has_rpm_spec: + lib.rpm_spec.add_depend(project, metapkg, conflict_version) + + lib.debian.changelog_add_entry_if_needed(project, feed, version_epoch) + + os.makedirs(lib.get_output_path(project)) + lib.remove_cache_extra_files() + + project_specific_func = f"prepare_project_{os.path.basename(project)}" + if project_specific_func in globals(): + print(f"{project}: running {project_specific_func}") + globals()[project_specific_func]() + + lib.debian.build_source_package(project) + lib.debian.move_files_to_output(project) + + if has_rpm_spec: + lib.rpm_spec.generate(project, version, epoch) + lib.rpm_spec.copy_to_output(project) + + lib.remove_cache_extra_files() + return version_epoch diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py new file mode 100755 index 0000000..759acfe --- /dev/null +++ b/scripts/obs/update_obs_project.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import argparse +import os +import traceback +import lib +import lib.config +import lib.docker +import lib.git +import lib.metapkg +import lib.osc +import lib.srcpkg + +srcpkgs_built = {} # dict of pkgname: version +srcpkgs_skipped = [] # list of pkgnames +srcpkgs_failed_build = [] # list of pkgnames +srcpkgs_failed_upload = [] # list of pkgnames +srcpkgs_updated = [] # list of pkgnames + + +def parse_packages(packages_arg): + if packages_arg: + for package in packages_arg: + lib.check_package(package) + return packages_arg + + # Default to all + ret = [] + ret += lib.config.projects_osmocom + ret += lib.config.projects_other + return ret + + +def build_srcpkg(feed, package, conflict_version, fetch, is_meta_pkg): + global srcpkgs_built + global srcpkgs_failed_build + + version = None + + try: + if is_meta_pkg: + version = lib.metapkg.build(feed, conflict_version) + else: + version = lib.srcpkg.build(package, feed, conflict_version, fetch) + srcpkgs_built[package] = version + except Exception as ex: + traceback.print_exception(type(ex), ex, ex.__traceback__) + print() + print(f"{package}: build failed") + srcpkgs_failed_build += [package] + + +def is_up_to_date(obs_version, git_latest_version): + if obs_version == git_latest_version: + return True + + # e.g. open5gs has "v" infront of version in git tag + if f"v{obs_version}" == git_latest_version: + return True + + return False + + +def build_srcpkg_if_needed(proj, feed, pkgs_remote, package, conflict_version, + fetch, is_meta_pkg, skip_up_to_date): + global srcpkgs_skipped + + if feed != "latest": + print(f"{package}: building source package (feed is {feed})") + else: + if is_meta_pkg: + latest_version = conflict_version if conflict_version else "1.0.0" + else: + latest_version = lib.git.get_latest_tag_remote(package) + + if latest_version is None: + print(f"{package}: skipping (no git tag found)") + srcpkgs_skipped += [package] + return + + if os.path.basename(package) not in pkgs_remote: + print(f"{package}: building source package (not in OBS)") + else: + obs_version = lib.osc.get_package_version(proj, package) + if is_up_to_date(obs_version, latest_version): + if skip_up_to_date: + print(f"{package}: skipping ({obs_version} is up-to-date)") + srcpkgs_skipped += [package] + return + else: + print(f"{package}: building source package" + f" ({obs_version} is up-to-date, but" + " --no-skip-up-to-date is set)") + else: + print(f"{package}: building source package (outdated:" + f" {latest_version} <=> {obs_version} in OBS)") + + build_srcpkg(feed, package, conflict_version, fetch, is_meta_pkg) + + +def upload_srcpkg(proj, feed, pkgs_remote, package, version): + if os.path.basename(package) not in pkgs_remote: + lib.osc.create_package(proj, package) + lib.osc.update_package(proj, package, version) + + +def build_srcpkgs(proj, feed, pkgs_remote, packages, conflict_version, fetch, + meta, skip_up_to_date): + print() + print("### Building source packages ###") + print() + + if meta: + build_srcpkg_if_needed(proj, feed, pkgs_remote, f"osmocom-{feed}", + conflict_version, fetch, True, skip_up_to_date) + + for package in packages: + build_srcpkg_if_needed(proj, feed, pkgs_remote, package, + conflict_version, fetch, False, skip_up_to_date) + + +def upload_srcpkgs(proj, feed, pkgs_remote): + global srcpkgs_built + global srcpkgs_failed_upload + global srcpkgs_updated + + srcpkgs_failed_upload = [] + srcpkgs_updated = [] + + if not srcpkgs_built: + return + + print() + print("### Uploading built packages ###") + print() + + for package, version in srcpkgs_built.items(): + try: + upload_srcpkg(proj, feed, pkgs_remote, package, version) + srcpkgs_updated += [package] + except Exception as ex: + traceback.print_exception(type(ex), ex, ex.__traceback__) + print() + print(f"{package}: upload failed") + srcpkgs_failed_upload += [package] + + +def exit_with_summary(): + global srcpkgs_updated + global srcpkgs_skipped + global srcpkgs_failed_build + global srcpkgs_failed_upload + + print() + print("### Summary ###") + print() + print(f"Updated: {len(srcpkgs_updated)}") + print(f"Skipped: {len(srcpkgs_skipped)}") + print(f"Failed (srcpkg build): {len(srcpkgs_failed_build)}") + print(f"Failed (srcpkg upload): {len(srcpkgs_failed_upload)}") + + if not srcpkgs_failed_build and not srcpkgs_failed_upload: + exit(0) + + print() + print("List of failed packages:") + for package in srcpkgs_failed_build: + print(f"* {package} (srcpkg build)") + for package in srcpkgs_failed_upload: + print(f"* {package} (srcpkg upload)") + + exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Generate source packages and upload them to OBS.") + lib.add_shared_arguments(parser) + parser.add_argument("-A", "--apiurl", help="OBS API URL or .oscrc alias" + " (e.g. https://obs.osmocom.org)") + parser.add_argument("-n", "--no-skip-up-to-date", + dest="skip_up_to_date", action="store_false", + help="for latest feed, build and upload packages even" + " if the version did not change") + parser.add_argument("obs_project", + help="OBS project, e.g. home:osmith:nightly") + parser.add_argument("package", nargs="*", + help="package name, e.g. libosmocore or open5gs," + " default is all packages") + args = parser.parse_args() + proj = args.obs_project + feed = args.feed + packages = parse_packages(args.package) + + lib.set_cmds_verbose(args.verbose) + + if args.docker: + lib.docker.run_in_docker_and_exit(__file__, args, True) + + lib.osc.check_proj(proj) + lib.osc.set_apiurl(args.apiurl) + lib.check_required_programs() + lib.remove_temp() + + pkgs_remote = lib.osc.get_remote_pkgs(proj) + + build_srcpkgs(proj, feed, pkgs_remote, packages, args.conflict_version, + args.git_fetch, args.meta, args.skip_up_to_date) + upload_srcpkgs(proj, feed, pkgs_remote) + exit_with_summary() + + +if __name__ == "__main__": + main() diff --git a/scripts/osmocom-latest-packages.sh b/scripts/osmocom-latest-packages.sh deleted file mode 100755 index 9055f5c..0000000 --- a/scripts/osmocom-latest-packages.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/sh -# Generate source packages and upload them to OBS, for the latest feed. -# New packages are only uploaded if the source changed. -# Environment variables: -# * PROJ: the OBS namespace to upload to (e.g. "network:osmocom:latest") -# * FEED: -# * "latest": use latest tagged release (default) -# * other: use last commit of branch of same name, exit with error if it does -# not exist -# * PACKAGES: set to a space-separated list of packages to skip all others -. "$(dirname "$0")/common.sh" -. "$(dirname "$0")/common-obs.sh" - -# Values for FEED env var. Adjust FEEDS_ALL in common-obs when changing. -FEEDS=" - latest -" - -set -e -set -x - -DT=$(date +%Y%m%d%H%M) -TOP=$(pwd) -DEBSRCDIR="$TOP/debsrc" -FEED="${FEED:-latest}" - -### OBS build -prepare() { - # start with a checkout of the project - if [ -d $PROJ ]; then - (cd $PROJ && osc up) - else - osc co $PROJ - fi - [ -d "$DEBSRCDIR" ] && rm -rf "$DEBSRCDIR" - mkdir "$DEBSRCDIR" - - cd "$TOP" - osmo_obs_prepare_conflict -} - -get_last_tag() { - project="$1" - if [ "$project" = "limesuite" ] || [ "$project" = "open5gs" ]; then - ver_regexp="^v[0-9]*.[0-9]*.[0-9]*$" - else - ver_regexp="^[0-9]*.[0-9]*.[0-9]*$" - fi - VER=$(git -C "${TOP}/${project}" tag -l --sort=v:refname | grep "$ver_regexp" | tail -n 1) - echo "${VER}" -} - -checkout() { - path=$1 - project="$(basename $path)" - url=$2 - gitbpargs="" - - if osmo_obs_skip_pkg "$project"; then - return - fi - - if [ -z "$url" ]; then - url="$(osmo_git_clone_url "$path")" - fi - - echo - echo "====> Checking out $project" - cd "$TOP" - [ -d "$project" ] || osmo_git_clone_date "$url" "$project" - cd "$project" - git fetch - - if [ "$FEED" = "latest" ]; then - VER=$(get_last_tag "$project") - git checkout -f -B "$VER" "refs/tags/$VER" - else - git checkout -f -B "$FEED" "origin/$FEED" - fi - - if [ "$project" = "open5gs" ]; then - meson subprojects download freeDiameter - fi -} - -# Generate a source package and upload it to OBS -# $1: package name (e.g. "libosmocore") -# $2: arguments to pass to "gbp buildpackage" -build() { - project=$1 - gitbpargs="$2" - output="$DEBSRCDIR/$project" - - if osmo_obs_skip_pkg "$project"; then - return - fi - - echo - echo "====> Building $project" - cd "$TOP/$project" - - osmo_obs_git_version_gen - - if [ "$FEED" = "latest" ]; then - debian_branch=$(get_last_tag "$project") - else - debian_branch="$FEED" - # Set new debian changelog version with commit appended. This version will - # become part of resulting filenames, and will change if commits have been - # added to the feed's branch. - VER="$(osmo_obs_get_commit_version)" - dch -b -v "$VER" -m "Snapshot build" - git commit -m "$VER snapshot" debian/ - fi - - osmo_obs_add_depend_deb "./debian/control" "$project" "osmocom-$FEED" - - if [ "$project" = "open5gs" ]; then - # we cannot control the output directory of the generated source :( - dpkg-buildpackage -S -uc -us -d - mkdir -p "$output" - mv "../$name"*.tar* "../$name"*.dsc "$output" - elif [ -x ./git-version-gen ]; then - gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$output" \ - "--git-debian-branch=$debian_branch" --git-ignore-new $gitbpargs \ - --git-postexport='cp $GBP_GIT_DIR/../.tarball-version $GBP_TMP_DIR/' - else - gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$output" \ - "--git-debian-branch=$debian_branch" --git-ignore-new $gitbpargs - fi - - if [ ! -d "$TOP/$PROJ/$project" ] ; then - # creating a new package is different from using old ones - mkdir "$TOP/$PROJ/$project" - mv "$output/"*.dsc "$TOP/$PROJ/$project/" - mv "$output/"*.tar* "$TOP/$PROJ/$project/" - cd "$TOP/$PROJ" - osc add "$project" - else - cd "$TOP/$PROJ/$project" - - # update OBS only if the filename doesn't match - file=$(cd "$output/" ; ls ./*.dsc) - if [ ! -e "$file" ] ; then - osc rm ./* || true - mv "$output/"*.dsc . - mv "$output/"*.tar* . - osc add ./* - fi - fi - - osmo_obs_add_rpm_spec "$TOP/$PROJ/$project" "$TOP/$project" "$project" "osmocom-$FEED" - - cd "$TOP" -} - -# add those once they have tagged any versions that include the 'debian' sub-dir: -#rtl-sdr -#osmo-fl2k - -build_osmocom() { - prepare - - # NOTE: when adding a repository that is not in gerrit, adjust osmo_git_clone_url() - checkout limesuite https://github.com/myriadrf/LimeSuite - checkout osmo-gsm-manuals - checkout libosmocore - checkout libosmo-sccp - checkout libosmo-abis - checkout libosmo-netif - checkout libsmpp34 - checkout libasn1c - checkout libgtpnl - checkout libusrp - checkout osmo-iuh - checkout osmo-ggsn - checkout osmo-sgsn - checkout osmo-pcap - checkout osmo-trx - checkout osmo-sip-connector - checkout osmo-bts - checkout osmo-pcu - checkout osmo-hlr - checkout osmo-mgw - checkout osmo-msc - checkout osmo-bsc - checkout python/osmo-python-tests - checkout simtrace2 - checkout libosmo-dsp - checkout osmo-sysmon - checkout osmo-remsim - checkout open5gs https://github.com/open5gs/open5gs - checkout neocon https://github.com/laf0rge/neocon - checkout osmo-uecups - checkout osmo-e1d - checkout osmo-smlc - checkout osmo-cbc - checkout osmo-gbproxy - checkout osmo-hnodeb - checkout osmo-hnbgw - - cd "$TOP" - - build osmocom-$FEED - build limesuite --git-upstream-tree="$(get_last_tag limesuite)" - build osmo-gsm-manuals - build libosmocore - build libosmo-sccp - build libosmo-abis - build libosmo-netif - build libsmpp34 - build libasn1c - build libgtpnl - build libusrp - build osmo-iuh - build osmo-ggsn - build osmo-sgsn - build osmo-pcap - build osmo-trx - build osmo-sip-connector - build osmo-bts - build osmo-pcu - build osmo-hlr - build osmo-mgw - build osmo-msc - build osmo-bsc - build osmo-python-tests - build simtrace2 - build libosmo-dsp - build osmo-sysmon - build osmo-remsim - build open5gs - build neocon - build osmo-uecups - build osmo-e1d - build osmo-smlc - build osmo-cbc - build osmo-gbproxy - build osmo-hnodeb - build osmo-hnbgw - - cd "$TOP/$PROJ" - osc ci -m "$FEED versions of $DT" --noservice -} - -osmo_obs_verify_feed -build_osmocom diff --git a/scripts/osmocom-next-packages.sh b/scripts/osmocom-next-packages.sh deleted file mode 100755 index aa9ab90..0000000 --- a/scripts/osmocom-next-packages.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Generate source packages and upload them to OBS, for the next feed. -. "$(dirname "$0")/common.sh" - -export FEED="next" -$OSMO_CI_DIR/scripts/osmocom-nightly-packages.sh diff --git a/scripts/osmocom-nightly-packages.sh b/scripts/osmocom-nightly-packages.sh deleted file mode 100755 index 426495e..0000000 --- a/scripts/osmocom-nightly-packages.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -# Generate source packages and upload them to OBS, for the nightly or next feed. -# New packages are always uploaded, even if the source does not change. Only -# packages of the same build date (DT) can be installed together. -# Environment variables: -# * PROJ: the OBS namespace to upload to (e.g. "network:osmocom:nightly") -# * FEED: controls the source branch that is used: -# * "nightly": use "master" branch (default) -# * "next": use "next" branch if it exists, otherwise use "master" branch -# * other (e.g. "2021q1"): use last commit of branch of same name, exit with -# error if it does not exist -# * PACKAGES: set to a space-separated list of packages to skip all others -. "$(dirname "$0")/common.sh" -. "$(dirname "$0")/common-obs.sh" - -# Values for FEED env var. Adjust FEEDS_ALL in common-obs when changing. -FEEDS=" - 2021q1 - 2021q4 - 2022q1 - 2022q2 - next - nightly -" - -set -e -set -x - -DT=$(date +%Y%m%d%H%M) -OSMO_OBS_CONFLICT_PKGVER="$OSMO_OBS_CONFLICT_PKGVER.$DT" -TOP=$(pwd)/$(mktemp -d nightly-3g_XXXXXX) -FEED="${FEED:-nightly}" - -### OBS build -prepare() { - # clean up the whole space - mkdir -p "$REPO/osc/" - cd "$REPO/osc" - osc co "$PROJ" - - cd "$REPO" - osmo_obs_prepare_conflict -} - -get_last_tag() { - project="$1" - if [ "$project" = "limesuite" ]; then - ver_regexp="^v[0-9]*.[0-9]*.[0-9]*$" - else - ver_regexp="^[0-9]*.[0-9]*.[0-9]*$" - fi - VER=$(git -C "${REPO}/${project}" tag -l --sort=v:refname | grep "$ver_regexp" | tail -n 1) - echo "${VER}" -} - -### common -checkout() { - local name=$1 - local url=$2 - local branch=$3 - - if osmo_obs_skip_pkg "$name"; then - return - fi - - if [ -z "$url" ]; then - url="$(osmo_git_clone_url "$name")" - fi - - if [ -z "$branch" ]; then - case "$FEED" in - nightly|next) ;; - *) branch="$FEED" ;; # e.g. 2021q1 - esac - fi - - cd "$REPO" - - if [ -n "$branch" ] ; then - osmo_git_clone_date "$url" -b "$branch" - else - osmo_git_clone_date "$url" - fi - - if [ "$FEED" = "next" ] && git -C "$name" show-branch remotes/origin/next >/dev/null 2>&1; then - git -C "$name" checkout next - fi - - cd - -} - -# Generate a source package and upload it to OBS -# $1: package name (e.g. "libosmocore") -# $2: update debian dir, unless set to "no_commit": -# * add dependency on osmocom-$FEED package -# * add new version to changelog (e.g. "1.5.1.96.c96d7.202104281354") -# $3: arguments to pass to "gbp buildpackage" -build() { - local name=$1 - local changelog=$2 - local gitbpargs=$3 - local repodir=$REPO/$name - local oscdir=$REPO/osc/$PROJ/$name - local dependver="$OSMO_OBS_CONFLICT_PKGVER" - - if osmo_obs_skip_pkg "$name"; then - return - fi - - if [ -d "$oscdir" ] ; then - # remove earlier version - cd "$oscdir" - osc rm -- * || true - else - # new package - mkdir -p "$oscdir/" - cd "$REPO/osc/$PROJ/" - osc add "$name" - fi - - cd "$repodir" - - if [ "$changelog" != "no_commit" ] ; then - osmo_obs_git_version_gen - # Add date to increase version even if commit did not change (OS#5135) - VER="$(osmo_obs_get_commit_version).$DT" - osmo_obs_add_depend_deb "./debian/control" "$name" "osmocom-$FEED" "$dependver" - dch -b -v "$VER" -m "Snapshot build" - git commit -m "$VER snapshot" debian/ - fi - - mkdir -p "$DATA/$name" - if [ "$name" = "open5gs" ]; then - # we cannot control the output directory of the generated source :( - dpkg-buildpackage -S -uc -us -d - mv "../$name"*.tar* "../$name"*.dsc "$DATA/$name/" - elif [ -f .tarball-version ]; then - gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$DATA/$name" \ - --git-ignore-new $gitbpargs \ - --git-postexport='cp $GBP_GIT_DIR/../.tarball-version $GBP_TMP_DIR/' - else - gbp buildpackage -S -uc -us -d --git-ignore-branch "--git-export-dir=$DATA/$name" \ - --git-ignore-new $gitbpargs - fi - - mv "$DATA/$name/"*.tar* "$DATA/$name/"*.dsc "$oscdir/" - - cd "$oscdir" - osc add -- *.tar* *.dsc - osmo_obs_add_rpm_spec "$oscdir" "$repodir" "$name" "osmocom-$FEED" "$dependver" - osc ci -m "Snapshot $name $DT" --noservice -} - -post() { - cd "$REPO/osc/$PROJ/" - osc status -} - -checkout_limesuite() { - if osmo_obs_skip_pkg "limesuite"; then - return - fi - - cd "$REPO" - git clone https://github.com/myriadrf/LimeSuite limesuite - TAG="$(get_last_tag limesuite)" - cd limesuite - git checkout "$TAG" -} - -checkout_open5gs() { - if osmo_obs_skip_pkg "open5gs"; then - return - fi - - cd "$REPO" - git clone https://github.com/open5gs/open5gs - cd open5gs - meson subprojects download freeDiameter -} - -build_osmocom() { - DATA=$TOP/data - REPO=$TOP/repo - - # rather than including a dangerous 'rm -rf *' here, lets delegate to the user: - if [ -n "$(ls $TOP)" ]; then - echo "ERROR: I need to run in an empty directory." - exit 1 - fi - - prepare - - # NOTE: when adding a repository that is not in gerrit, adjust osmo_git_clone_url() - checkout_limesuite - checkout osmo-gsm-manuals - checkout libosmocore - checkout libosmo-sccp - checkout libosmo-abis - checkout libosmo-netif - checkout libsmpp34 - checkout libasn1c - checkout libgtpnl - checkout libusrp - checkout osmo-iuh - checkout osmo-ggsn - checkout osmo-sgsn - checkout osmo-pcap - checkout osmo-trx - checkout osmo-sip-connector - checkout osmo-bts - checkout osmo-pcu - checkout osmo-hlr - checkout osmo-mgw - checkout osmo-msc - checkout osmo-bsc - checkout python/osmo-python-tests - checkout rtl-sdr - checkout osmo-fl2k - checkout simtrace2 - checkout libosmo-dsp - checkout osmo-sysmon - checkout osmo-remsim - checkout_open5gs - checkout neocon https://github.com/laf0rge/neocon - checkout osmo-uecups - checkout osmo-e1d - checkout osmo-smlc - checkout osmo-cbc - checkout osmo-gbproxy - checkout osmo-hnodeb - checkout osmo-hnbgw - checkout osmo-bsc-nat - - cd "$REPO" - - build osmocom-$FEED no_commit - build limesuite no_commit --git-upstream-tree="$(get_last_tag limesuite)" - build osmo-gsm-manuals - build libosmocore - build libosmo-sccp - build libosmo-abis - build libosmo-netif - build libsmpp34 - build libasn1c - build libgtpnl - build libusrp - build osmo-iuh - build osmo-ggsn - build osmo-sgsn - build osmo-pcap - build osmo-trx - build osmo-sip-connector - build osmo-bts - build osmo-pcu - build osmo-hlr - build osmo-mgw - build osmo-msc - build osmo-bsc - build osmo-python-tests - build rtl-sdr - build osmo-fl2k - build simtrace2 - build libosmo-dsp - build osmo-sysmon - build osmo-remsim - build open5gs - build neocon - build osmo-uecups - build osmo-e1d - build osmo-smlc - build osmo-cbc - build osmo-gbproxy - build osmo-hnodeb - build osmo-hnbgw - build osmo-bsc-nat - - post -} - -osmo_obs_verify_feed -build_osmocom -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/28882 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I45a555d05a9da808c0fe0145aae665f583cb80d9 Gerrit-Change-Number: 28882 Gerrit-PatchSet: 1 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-MessageType: newchange

2 years, 10 months

1
0
0 0

2025

2024

2023

2022

gerrit-log August 2022