diff --git a/LICENSE b/LICENSE index 29ebfa5..4153cd3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,661 +1,287 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. \ No newline at end of file + EUROPEAN UNION PUBLIC LICENCE v. 1.2 + EUPL © the European Union 2007, 2016 + +This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined +below) which is provided under the terms of this Licence. Any use of the Work, +other than as authorised under this Licence is prohibited (to the extent such +use is covered by a right of the copyright holder of the Work). + +The Work is provided under the terms of this Licence when the Licensor (as +defined below) has placed the following notice immediately following the +copyright notice for the Work: + + Licensed under the EUPL + +or has expressed by any other means his willingness to license under the EUPL. + +1. Definitions + +In this Licence, the following terms have the following meaning: + +- ‘The Licence’: this Licence. + +- ‘The Original Work’: the work or software distributed or communicated by the + Licensor under this Licence, available as Source Code and also as Executable + Code as the case may be. + +- ‘Derivative Works’: the works or software that could be created by the + Licensee, based upon the Original Work or modifications thereof. This Licence + does not define the extent of modification or dependence on the Original Work + required in order to classify a work as a Derivative Work; this extent is + determined by copyright law applicable in the country mentioned in Article 15. + +- ‘The Work’: the Original Work or its Derivative Works. + +- ‘The Source Code’: the human-readable form of the Work which is the most + convenient for people to study and modify. + +- ‘The Executable Code’: any code which has generally been compiled and which is + meant to be interpreted by a computer as a program. + +- ‘The Licensor’: the natural or legal person that distributes or communicates + the Work under the Licence. + +- ‘Contributor(s)’: any natural or legal person who modifies the Work under the + Licence, or otherwise contributes to the creation of a Derivative Work. + +- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of + the Work under the terms of the Licence. + +- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, + renting, distributing, communicating, transmitting, or otherwise making + available, online or offline, copies of the Work or providing access to its + essential functionalities at the disposal of any other natural or legal + person. + +2. Scope of the rights granted by the Licence + +The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, +sublicensable licence to do the following, for the duration of copyright vested +in the Original Work: + +- use the Work in any circumstance and for all usage, +- reproduce the Work, +- modify the Work, and make Derivative Works based upon the Work, +- communicate to the public, including the right to make available or display + the Work or copies thereof to the public and perform publicly, as the case may + be, the Work, +- distribute the Work or copies thereof, +- lend and rent the Work or copies thereof, +- sublicense rights in the Work or copies thereof. + +Those rights can be exercised on any media, supports and formats, whether now +known or later invented, as far as the applicable law permits so. + +In the countries where moral rights apply, the Licensor waives his right to +exercise his moral right to the extent allowed by law in order to make effective +the licence of the economic rights here above listed. + +The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to +any patents held by the Licensor, to the extent necessary to make use of the +rights granted on the Work under this Licence. + +3. Communication of the Source Code + +The Licensor may provide the Work either in its Source Code form, or as +Executable Code. If the Work is provided as Executable Code, the Licensor +provides in addition a machine-readable copy of the Source Code of the Work +along with each copy of the Work that the Licensor distributes or indicates, in +a notice following the copyright notice attached to the Work, a repository where +the Source Code is easily and freely accessible for as long as the Licensor +continues to distribute or communicate the Work. + +4. Limitations on copyright + +Nothing in this Licence is intended to deprive the Licensee of the benefits from +any exception or limitation to the exclusive rights of the rights owners in the +Work, of the exhaustion of those rights or of other applicable limitations +thereto. + +5. Obligations of the Licensee + +The grant of the rights mentioned above is subject to some restrictions and +obligations imposed on the Licensee. Those obligations are the following: + +Attribution right: The Licensee shall keep intact all copyright, patent or +trademarks notices and all notices that refer to the Licence and to the +disclaimer of warranties. The Licensee must include a copy of such notices and a +copy of the Licence with every copy of the Work he/she distributes or +communicates. The Licensee must cause any Derivative Work to carry prominent +notices stating that the Work has been modified and the date of modification. + +Copyleft clause: If the Licensee distributes or communicates copies of the +Original Works or Derivative Works, this Distribution or Communication will be +done under the terms of this Licence or of a later version of this Licence +unless the Original Work is expressly distributed only under this version of the +Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee +(becoming Licensor) cannot offer or impose any additional terms or conditions on +the Work or Derivative Work that alter or restrict the terms of the Licence. + +Compatibility clause: If the Licensee Distributes or Communicates Derivative +Works or copies thereof based upon both the Work and another work licensed under +a Compatible Licence, this Distribution or Communication can be done under the +terms of this Compatible Licence. For the sake of this clause, ‘Compatible +Licence’ refers to the licences listed in the appendix attached to this Licence. +Should the Licensee's obligations under the Compatible Licence conflict with +his/her obligations under this Licence, the obligations of the Compatible +Licence shall prevail. + +Provision of Source Code: When distributing or communicating copies of the Work, +the Licensee will provide a machine-readable copy of the Source Code or indicate +a repository where this Source will be easily and freely available for as long +as the Licensee continues to distribute or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade names, +trademarks, service marks, or names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the copyright notice. + +6. Chain of Authorship + +The original Licensor warrants that the copyright in the Original Work granted +hereunder is owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each Contributor warrants that the copyright in the modifications he/she brings +to the Work are owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each time You accept the Licence, the original Licensor and subsequent +Contributors grant You a licence to their contributions to the Work, under the +terms of this Licence. + +7. Disclaimer of Warranty + +The Work is a work in progress, which is continuously improved by numerous +Contributors. It is not a finished work and may therefore contain defects or +‘bugs’ inherent to this type of development. + +For the above reason, the Work is provided under the Licence on an ‘as is’ basis +and without warranties of any kind concerning the Work, including without +limitation merchantability, fitness for a particular purpose, absence of defects +or errors, accuracy, non-infringement of intellectual property rights other than +copyright as stated in Article 6 of this Licence. + +This disclaimer of warranty is an essential part of the Licence and a condition +for the grant of any rights to the Work. + +8. Disclaimer of Liability + +Except in the cases of wilful misconduct or damages directly caused to natural +persons, the Licensor will in no event be liable for any direct or indirect, +material or moral, damages of any kind, arising out of the Licence or of the use +of the Work, including without limitation, damages for loss of goodwill, work +stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such damage. +However, the Licensor will be liable under statutory product liability laws as +far such laws apply to the Work. + +9. Additional agreements + +While distributing the Work, You may choose to conclude an additional agreement, +defining obligations or services consistent with this Licence. However, if +accepting obligations, You may act only on your own behalf and on your sole +responsibility, not on behalf of the original Licensor or any other Contributor, +and only if You agree to indemnify, defend, and hold each Contributor harmless +for any liability incurred by, or claims asserted against such Contributor by +the fact You have accepted any warranty or additional liability. + +10. Acceptance of the Licence + +The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ +placed under the bottom of a window displaying the text of this Licence or by +affirming consent in any other similar way, in accordance with the rules of +applicable law. Clicking on that icon indicates your clear and irrevocable +acceptance of this Licence and all of its terms and conditions. + +Similarly, you irrevocably accept this Licence and all of its terms and +conditions by exercising any rights granted to You by Article 2 of this Licence, +such as the use of the Work, the creation by You of a Derivative Work or the +Distribution or Communication by You of the Work or copies thereof. + +11. Information to the public + +In case of any Distribution or Communication of the Work by means of electronic +communication by You (for example, by offering to download the Work from a +remote location) the distribution channel or media (for example, a website) must +at least provide to the public the information requested by the applicable law +regarding the Licensor, the Licence and the way it may be accessible, concluded, +stored and reproduced by the Licensee. + +12. Termination of the Licence + +The Licence and the rights granted hereunder will terminate automatically upon +any breach by the Licensee of the terms of the Licence. + +Such a termination will not terminate the licences of any person who has +received the Work from the Licensee under the Licence, provided such persons +remain in full compliance with the Licence. + +13. Miscellaneous + +Without prejudice of Article 9 above, the Licence represents the complete +agreement between the Parties as to the Work. + +If any provision of the Licence is invalid or unenforceable under applicable +law, this will not affect the validity or enforceability of the Licence as a +whole. Such provision will be construed or reformed so as necessary to make it +valid and enforceable. + +The European Commission may publish other linguistic versions or new versions of +this Licence or updated versions of the Appendix, so far this is required and +reasonable, without reducing the scope of the rights granted by the Licence. New +versions of the Licence will be published with a unique version number. + +All linguistic versions of this Licence, approved by the European Commission, +have identical value. Parties can take advantage of the linguistic version of +their choice. + +14. Jurisdiction + +Without prejudice to specific agreement between parties, + +- any litigation resulting from the interpretation of this License, arising + between the European Union institutions, bodies, offices or agencies, as a + Licensor, and any Licensee, will be subject to the jurisdiction of the Court + of Justice of the European Union, as laid down in article 272 of the Treaty on + the Functioning of the European Union, + +- any litigation arising between other parties and resulting from the + interpretation of this License, will be subject to the exclusive jurisdiction + of the competent court where the Licensor resides or conducts its primary + business. + +15. Applicable Law + +Without prejudice to specific agreement between parties, + +- this Licence shall be governed by the law of the European Union Member State + where the Licensor has his seat, resides or has his registered office, + +- this licence shall be governed by Belgian law if the Licensor has no seat, + residence or registered office inside a European Union Member State. + +Appendix + +‘Compatible Licences’ according to Article 5 EUPL are: + +- GNU General Public License (GPL) v. 2, v. 3 +- GNU Affero General Public License (AGPL) v. 3 +- Open Software License (OSL) v. 2.1, v. 3.0 +- Eclipse Public License (EPL) v. 1.0 +- CeCILL v. 2.0, v. 2.1 +- Mozilla Public Licence (MPL) v. 2 +- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for + works other than software +- European Union Public Licence (EUPL) v. 1.1, v. 1.2 +- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong + Reciprocity (LiLiQ-R+). + +The European Commission may update this Appendix to later versions of the above +licences without producing a new version of the EUPL, as long as they provide +the rights granted in Article 2 of this Licence and protect the covered Source +Code from exclusive appropriation. + +All other changes or additions to this Appendix require the production of a new +EUPL version. diff --git a/Source/Directory.Packages.props b/Source/Directory.Packages.props index a066f05..d6b666b 100644 --- a/Source/Directory.Packages.props +++ b/Source/Directory.Packages.props @@ -24,12 +24,13 @@ - + + diff --git a/Source/Reef.Tests/Scripting/InterpreterServiceTests.cs b/Source/Reef.Tests/Scripting/InterpreterServiceTests.cs new file mode 100644 index 0000000..328d5d5 --- /dev/null +++ b/Source/Reef.Tests/Scripting/InterpreterServiceTests.cs @@ -0,0 +1,51 @@ +using FluentAssertions; +using Reef.Core.Scripting; +using Reef.Core.Services; + +namespace Reef.Tests.Scripting; + +public class InterpreterServiceTests +{ + private readonly InterpreterService _service = new(new ProcessScriptRunner()); + + [Fact] + public async Task CheckOneAsync_RunnableInterpreter_ReportsAvailableWithOutput() + { + var status = await _service.CheckOneAsync("sh"); + + status.Available.Should().BeTrue(); + status.Output.Should().Be("ok"); + status.Error.Should().BeNull(); + } + + [Fact] + public async Task CheckOneAsync_UnknownInterpreter_ReportsUnavailableWithError() + { + var status = await _service.CheckOneAsync("not-a-real-interpreter"); + + status.Available.Should().BeFalse(); + status.Error.Should().Contain("Unknown interpreter"); + } + + [Fact] + public async Task CheckAllAsync_ReturnsOneStatusPerKnownInterpreter() + { + var statuses = await _service.CheckAllAsync(); + + statuses.Should().HaveCount(InterpreterService.KnownInterpreters.Count); + statuses.Select(s => s.Name).Should().BeEquivalentTo( + InterpreterService.KnownInterpreters.Select(d => d.Name)); + } + + [Fact] + public async Task CheckAllAsync_OnThisHost_FindsAtLeastShAndBashAvailable() + { + // sh/bash are present on every Linux/macOS CI runner and dev box this + // project targets - a sane baseline so a regression in the probe + // logic itself doesn't silently report everything as unavailable. + var statuses = await _service.CheckAllAsync(); + + statuses.First(s => s.Name == "sh").Available.Should().BeTrue(); + statuses.First(s => s.Name == "bash").Available.Should().BeTrue(); + } +} diff --git a/Source/Reef.Tests/Scripting/ProcessScriptRunnerTests.cs b/Source/Reef.Tests/Scripting/ProcessScriptRunnerTests.cs new file mode 100644 index 0000000..28b5784 --- /dev/null +++ b/Source/Reef.Tests/Scripting/ProcessScriptRunnerTests.cs @@ -0,0 +1,129 @@ +using FluentAssertions; +using Reef.Core.Scripting; + +namespace Reef.Tests.Scripting; + +public class ProcessScriptRunnerTests +{ + private readonly ProcessScriptRunner _runner = new(); + + [Fact] + public async Task RunAsync_InlineScriptExitingZero_ReturnsSuccessWithStdout() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "echo hello-from-script", + IsInline = true, + StdinJson = "{}" + }); + + result.Success.Should().BeTrue(); + result.ExitCode.Should().Be(0); + result.Stdout.Should().Contain("hello-from-script"); + } + + [Fact] + public async Task RunAsync_InlineScriptExitingNonZero_ReturnsFailure() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "exit 7", + IsInline = true, + StdinJson = "{}" + }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().Be(7); + } + + [Fact] + public async Task RunAsync_ScriptReadsStdin_ReceivesPayload() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "cat", + IsInline = true, + StdinJson = "{\"executionId\":42}" + }); + + result.Success.Should().BeTrue(); + result.Stdout.Should().Contain("\"executionId\":42"); + } + + [Fact] + public async Task RunAsync_ScriptIgnoresStdin_DoesNotFailOnBrokenPipe() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "echo done", + IsInline = true, + StdinJson = new string('x', 1024 * 1024) // large payload the script never reads + }); + + result.Success.Should().BeTrue(); + result.Stdout.Should().Contain("done"); + } + + [Fact] + public async Task RunAsync_ScriptExceedsTimeout_IsKilledAndReportsTimeout() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "sleep 30", + IsInline = true, + StdinJson = "{}", + TimeoutSeconds = 1 + }); + + result.Success.Should().BeFalse(); + result.ErrorMessage.Should().Contain("timed out"); + } + + [Fact] + public async Task RunAsync_NonExistentScriptPath_ReturnsFailureWithoutThrowing() + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "/no/such/script.sh", + IsInline = false, + StdinJson = "{}" + }); + + result.Success.Should().BeFalse(); + result.ErrorMessage.Should().Contain("not found"); + } + + [Fact] + public async Task RunAsync_EnvAllowlist_PassesOnlyAllowedVariables() + { + Environment.SetEnvironmentVariable("REEF_TEST_ALLOWED", "visible"); + Environment.SetEnvironmentVariable("REEF_TEST_BLOCKED", "secret"); + + try + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "sh", + ScriptPathOrInline = "echo \"A=$REEF_TEST_ALLOWED B=$REEF_TEST_BLOCKED\"", + IsInline = true, + StdinJson = "{}", + EnvAllowlist = new List { "REEF_TEST_ALLOWED" } + }); + + result.Stdout.Should().Contain("A=visible"); + result.Stdout.Should().Contain("B="); + result.Stdout.Should().NotContain("secret"); + } + finally + { + Environment.SetEnvironmentVariable("REEF_TEST_ALLOWED", null); + Environment.SetEnvironmentVariable("REEF_TEST_BLOCKED", null); + } + } +} diff --git a/Source/Reef.Tests/Templates/MockHttpServer.cs b/Source/Reef.Tests/Templates/MockHttpServer.cs new file mode 100644 index 0000000..bd83976 --- /dev/null +++ b/Source/Reef.Tests/Templates/MockHttpServer.cs @@ -0,0 +1,96 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace Reef.Tests.Templates; + +/// +/// Minimal loopback HTTP server for exercising the example Script Templates +/// against a fake webhook/Slack/ERP endpoint instead of the real internet. +/// +public sealed class MockHttpServer : IDisposable +{ + private readonly HttpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _acceptLoop; + + private int _statusCode = 200; + private string _responseBody = "ok"; + + public string BaseUrl { get; } + public string? LastRequestBody { get; private set; } + public string? LastMethod { get; private set; } + public string? LastAuthorizationHeader { get; private set; } + public string? LastContentTypeHeader { get; private set; } + public int RequestCount { get; private set; } + + public MockHttpServer() + { + var port = GetFreePort(); + BaseUrl = $"http://127.0.0.1:{port}/"; + _listener = new HttpListener(); + _listener.Prefixes.Add(BaseUrl); + _listener.Start(); + _acceptLoop = Task.Run(AcceptLoopAsync); + } + + public void SetResponse(int statusCode, string body = "ok") + { + _statusCode = statusCode; + _responseBody = body; + } + + private async Task AcceptLoopAsync() + { + while (!_cts.IsCancellationRequested && _listener.IsListening) + { + HttpListenerContext ctx; + try + { + ctx = await _listener.GetContextAsync(); + } + catch + { + return; + } + + try + { + using var reader = new StreamReader(ctx.Request.InputStream, Encoding.UTF8); + LastRequestBody = await reader.ReadToEndAsync(); + LastMethod = ctx.Request.HttpMethod; + LastAuthorizationHeader = ctx.Request.Headers["Authorization"]; + LastContentTypeHeader = ctx.Request.Headers["Content-Type"]; + RequestCount++; + + ctx.Response.StatusCode = _statusCode; + var bytes = Encoding.UTF8.GetBytes(_responseBody); + ctx.Response.ContentLength64 = bytes.Length; + await ctx.Response.OutputStream.WriteAsync(bytes); + ctx.Response.OutputStream.Close(); + } + catch + { + // Best effort - a flaky/aborted request shouldn't kill the accept loop. + } + } + } + + private static int GetFreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + public void Dispose() + { + _cts.Cancel(); + _listener.Stop(); + _listener.Close(); + try { _acceptLoop.Wait(TimeSpan.FromSeconds(2)); } catch { /* best effort */ } + _cts.Dispose(); + } +} diff --git a/Source/Reef.Tests/Templates/ScriptTemplateExamplesTests.cs b/Source/Reef.Tests/Templates/ScriptTemplateExamplesTests.cs new file mode 100644 index 0000000..8053470 --- /dev/null +++ b/Source/Reef.Tests/Templates/ScriptTemplateExamplesTests.cs @@ -0,0 +1,428 @@ +using System.Text.Json; +using FluentAssertions; +using Reef.Core.Scripting; + +namespace Reef.Tests.Templates; + +/// +/// Exercises the example Script Templates shipped in /Templates against a +/// mock HTTP endpoint instead of real Slack/n8n/ERP services, verifying: +/// - the stdin JSON contract (PascalCase keys, matches ExecutionService's +/// RunProcessingScriptAsync payload shape) is what each script expects +/// - success/failure HTTP responses map to the right process exit code +/// - missing required env vars fail fast with a non-zero exit +/// These run the real script files via ProcessScriptRunner - the same path +/// production uses - not a reimplementation of their logic. +/// +public class ScriptTemplateExamplesTests +{ + private readonly ProcessScriptRunner _runner = new(); + private static readonly string TemplatesDir = FindTemplatesDir(); + + private static string FindTemplatesDir() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null) + { + var candidate = Path.Combine(dir.FullName, "Templates"); + if (Directory.Exists(candidate) && + File.Exists(Path.Combine(candidate, "Script_-_Webhook_n8n_Delivery.ScriptTemplate.sh"))) + { + return candidate; + } + dir = dir.Parent; + } + throw new DirectoryNotFoundException("Could not locate repository /Templates directory from test output path"); + } + + private static string SampleStdinJson(string status = "Success", int rowCount = 25) => + JsonSerializer.Serialize(new + { + ExecutionId = 1234, + ProfileId = 7, + RowCount = rowCount, + OutputPath = "/data/exports/orders.csv", + FileSizeBytes = 88210, + ExecutionTimeMs = 842, + OutputFormat = "CSV", + TriggeredBy = "Test", + StartedAt = DateTime.UtcNow, + CompletedAt = DateTime.UtcNow, + Status = status, + ErrorMessage = (string?)null + }); + + // Webhook / n8n (sh + curl) + + [Fact] + public async Task WebhookTemplate_SuccessResponse_ExitsZeroAndForwardsContext() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200, "{\"received\":true}"); + Environment.SetEnvironmentVariable("N8N_WEBHOOK_URL", mock.BaseUrl); + + try + { + var result = await RunTemplate("Script_-_Webhook_n8n_Delivery.ScriptTemplate.sh", "sh", + SampleStdinJson(), new List { "N8N_WEBHOOK_URL" }); + + result.Success.Should().BeTrue(); + result.ExitCode.Should().Be(0); + mock.RequestCount.Should().Be(1); + mock.LastMethod.Should().Be("POST"); + mock.LastContentTypeHeader.Should().Be("application/json"); + mock.LastRequestBody.Should().Contain("\"Status\":\"Success\""); + } + finally + { + Environment.SetEnvironmentVariable("N8N_WEBHOOK_URL", null); + } + } + + [Fact] + public async Task WebhookTemplate_ServerErrorResponse_FailsInsteadOfReportingSuccess() + { + // Regression test for the curl -f fix: without -f, curl exits 0 even on a 5xx, + // which would make Reef report "Success" while the webhook delivery silently failed. + using var mock = new MockHttpServer(); + mock.SetResponse(500, "internal error"); + Environment.SetEnvironmentVariable("N8N_WEBHOOK_URL", mock.BaseUrl); + + try + { + var result = await RunTemplate("Script_-_Webhook_n8n_Delivery.ScriptTemplate.sh", "sh", + SampleStdinJson(), new List { "N8N_WEBHOOK_URL" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + mock.RequestCount.Should().Be(1); + } + finally + { + Environment.SetEnvironmentVariable("N8N_WEBHOOK_URL", null); + } + } + + [Fact] + public async Task WebhookTemplate_MissingEnvVar_FailsFastWithoutCallingServer() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200); + Environment.SetEnvironmentVariable("N8N_WEBHOOK_URL", null); + + var result = await RunTemplate("Script_-_Webhook_n8n_Delivery.ScriptTemplate.sh", "sh", + SampleStdinJson(), new List { "N8N_WEBHOOK_URL" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + mock.RequestCount.Should().Be(0); + } + + // Slack / Teams notify (python) + + [Fact] + public async Task SlackTemplate_SuccessResponse_ExitsZeroAndPostsSummaryText() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200, "ok"); + Environment.SetEnvironmentVariable("SLACK_WEBHOOK_URL", mock.BaseUrl); + + try + { + var result = await RunTemplate("Script_-_Slack_Teams_Notify.ScriptTemplate.py", "python", + SampleStdinJson(rowCount: 250), new List { "SLACK_WEBHOOK_URL" }); + + result.Success.Should().BeTrue(); + result.ExitCode.Should().Be(0); + mock.LastMethod.Should().Be("POST"); + mock.LastRequestBody.Should().Contain("\"text\""); + mock.LastRequestBody.Should().Contain("250 rows"); + } + finally + { + Environment.SetEnvironmentVariable("SLACK_WEBHOOK_URL", null); + } + } + + [Fact] + public async Task SlackTemplate_ServerErrorResponse_Fails() + { + using var mock = new MockHttpServer(); + mock.SetResponse(503, "unavailable"); + Environment.SetEnvironmentVariable("SLACK_WEBHOOK_URL", mock.BaseUrl); + + try + { + var result = await RunTemplate("Script_-_Slack_Teams_Notify.ScriptTemplate.py", "python", + SampleStdinJson(), new List { "SLACK_WEBHOOK_URL" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + } + finally + { + Environment.SetEnvironmentVariable("SLACK_WEBHOOK_URL", null); + } + } + + [Fact] + public async Task SlackTemplate_MissingEnvVars_FailsFastWithoutCallingServer() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200); + Environment.SetEnvironmentVariable("SLACK_WEBHOOK_URL", null); + Environment.SetEnvironmentVariable("TEAMS_WEBHOOK_URL", null); + + var result = await RunTemplate("Script_-_Slack_Teams_Notify.ScriptTemplate.py", "python", + SampleStdinJson(), new List { "SLACK_WEBHOOK_URL", "TEAMS_WEBHOOK_URL" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + mock.RequestCount.Should().Be(0); + } + + // REST ERP call (node) + + [Fact] + public async Task ErpTemplate_SuccessResponse_PostsExecutionSummaryWithAuth() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200, "{\"accepted\":true}"); + Environment.SetEnvironmentVariable("ERP_API_URL", mock.BaseUrl); + Environment.SetEnvironmentVariable("ERP_API_KEY", "test-key-123"); + + try + { + var result = await RunTemplate("Script_-_REST_ERP_Call.ScriptTemplate.js", "node", + SampleStdinJson(), new List { "ERP_API_URL", "ERP_API_KEY" }); + + result.Success.Should().BeTrue(); + result.ExitCode.Should().Be(0); + mock.LastMethod.Should().Be("POST"); + mock.LastAuthorizationHeader.Should().Be("Bearer test-key-123"); + mock.LastRequestBody.Should().Contain("\"externalReference\":\"reef-execution-1234\""); + mock.LastRequestBody.Should().Contain("\"status\":\"Success\""); + } + finally + { + Environment.SetEnvironmentVariable("ERP_API_URL", null); + Environment.SetEnvironmentVariable("ERP_API_KEY", null); + } + } + + [Fact] + public async Task ErpTemplate_ServerErrorResponse_Fails() + { + using var mock = new MockHttpServer(); + mock.SetResponse(422, "validation failed"); + Environment.SetEnvironmentVariable("ERP_API_URL", mock.BaseUrl); + Environment.SetEnvironmentVariable("ERP_API_KEY", "test-key-123"); + + try + { + var result = await RunTemplate("Script_-_REST_ERP_Call.ScriptTemplate.js", "node", + SampleStdinJson(), new List { "ERP_API_URL", "ERP_API_KEY" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + } + finally + { + Environment.SetEnvironmentVariable("ERP_API_URL", null); + Environment.SetEnvironmentVariable("ERP_API_KEY", null); + } + } + + [Fact] + public async Task ErpTemplate_MissingEnvVars_FailsFastWithoutCallingServer() + { + using var mock = new MockHttpServer(); + mock.SetResponse(200); + Environment.SetEnvironmentVariable("ERP_API_URL", null); + Environment.SetEnvironmentVariable("ERP_API_KEY", null); + + var result = await RunTemplate("Script_-_REST_ERP_Call.ScriptTemplate.js", "node", + SampleStdinJson(), new List { "ERP_API_URL", "ERP_API_KEY" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + mock.RequestCount.Should().Be(0); + } + + // ExactXML (pwsh) - only runs where pwsh is actually installed; this sandbox + // doesn't have it, and the script also needs a real SQL Server staging table + // and ExactXML.exe binary, neither of which are mockable cheaply. We only + // verify the fast-fail-on-missing-config path, which needs no external deps. + + [Fact] + public async Task ExactXmlTemplate_MissingEnvVars_FailsFast() + { + if (!IsPwshAvailable()) + { + return; // pwsh not installed on this host - nothing to verify here + } + + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", null); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", null); + + var result = await RunTemplate("Script_-_REST_Import_to_ExactXML.ScriptTemplate.ps1", "pwsh", + JsonSerializer.Serialize(new { ExecutionId = 1 }), + new List { "STAGING_DB_CONNSTRING", "EXACTXML_EXE_PATH" }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + } + + // Bash equivalent of the ExactXML pattern (env-var-gated external exe call, + // exit code propagated to Reef) for hosts without pwsh. ProcessScriptRunner + // treats `bash` identically to `sh`/`pwsh` - same interpreter resolution, + // same inline-script-to-temp-file handling, same env allowlist and exit + // code propagation (see ProcessScriptRunner.ResolveInterpreter) - so this + // proves the pattern is fully exercisable on any host, not just Windows. + private const string BashExactXmlHarness = """ + #!/bin/bash + set -eu + : "$(cat)" # drain stdin, matching the real template's contract + if [ -z "${STAGING_DB_CONNSTRING:-}" ] || [ -z "${EXACTXML_EXE_PATH:-}" ]; then + echo "STAGING_DB_CONNSTRING and EXACTXML_EXE_PATH must be set" >&2 + exit 1 + fi + "$EXACTXML_EXE_PATH" + """; + + [Fact] + public async Task BashExactXmlHarness_MissingEnvVars_FailsFast() + { + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", null); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", null); + + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "bash", + ScriptPathOrInline = BashExactXmlHarness, + IsInline = true, + StdinJson = JsonSerializer.Serialize(new { ExecutionId = 1 }), + TimeoutSeconds = 10, + EnvAllowlist = new List { "STAGING_DB_CONNSTRING", "EXACTXML_EXE_PATH" } + }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().NotBe(0); + result.Stderr.Should().Contain("must be set"); + } + + [Fact] + public async Task BashExactXmlHarness_FakeExeSucceeds_PropagatesExitZero() + { + var fakeExe = WriteFakeExactXmlExe(exitCode: 0, stdout: "Imported 3 row(s)"); + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", "Server=fake;Database=fake;"); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", fakeExe); + + try + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "bash", + ScriptPathOrInline = BashExactXmlHarness, + IsInline = true, + StdinJson = JsonSerializer.Serialize(new { ExecutionId = 1 }), + TimeoutSeconds = 10, + EnvAllowlist = new List { "STAGING_DB_CONNSTRING", "EXACTXML_EXE_PATH" } + }); + + result.Success.Should().BeTrue(); + result.ExitCode.Should().Be(0); + result.Stdout.Should().Contain("Imported 3 row(s)"); + } + finally + { + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", null); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", null); + File.Delete(fakeExe); + } + } + + [Fact] + public async Task BashExactXmlHarness_FakeExeFails_PropagatesExitCode() + { + var fakeExe = WriteFakeExactXmlExe(exitCode: 7, stdout: "", stderr: "import definition mismatch"); + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", "Server=fake;Database=fake;"); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", fakeExe); + + try + { + var result = await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = "bash", + ScriptPathOrInline = BashExactXmlHarness, + IsInline = true, + StdinJson = JsonSerializer.Serialize(new { ExecutionId = 1 }), + TimeoutSeconds = 10, + EnvAllowlist = new List { "STAGING_DB_CONNSTRING", "EXACTXML_EXE_PATH" } + }); + + result.Success.Should().BeFalse(); + result.ExitCode.Should().Be(7); + result.Stderr.Should().Contain("import definition mismatch"); + } + finally + { + Environment.SetEnvironmentVariable("STAGING_DB_CONNSTRING", null); + Environment.SetEnvironmentVariable("EXACTXML_EXE_PATH", null); + File.Delete(fakeExe); + } + } + + /// Writes a tiny executable bash script standing in for ExactXML.exe. + private static string WriteFakeExactXmlExe(int exitCode, string stdout, string stderr = "") + { + var path = Path.Combine(Path.GetTempPath(), $"fake-exactxml-{Guid.NewGuid():N}.sh"); + var script = $""" + #!/bin/bash + {(string.IsNullOrEmpty(stdout) ? "" : $"echo \"{stdout}\"")} + {(string.IsNullOrEmpty(stderr) ? "" : $"echo \"{stderr}\" >&2")} + exit {exitCode} + """; + File.WriteAllText(path, script); + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + return path; + } + + private static bool IsPwshAvailable() + { + try + { + using var p = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo + { + FileName = "pwsh", + Arguments = "-NoProfile -Command exit", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }); + p?.WaitForExit(2000); + return p?.ExitCode == 0; + } + catch + { + return false; + } + } + + private async Task RunTemplate( + string fileName, string interpreter, string stdinJson, List envAllowlist) + { + var path = Path.Combine(TemplatesDir, fileName); + File.Exists(path).Should().BeTrue($"expected template file at {path}"); + + return await _runner.RunAsync(new ScriptExecutionRequest + { + Interpreter = interpreter, + ScriptPathOrInline = path, + IsInline = false, + StdinJson = stdinJson, + TimeoutSeconds = 15, + EnvAllowlist = envAllowlist + }); + } +} diff --git a/Source/Reef/Api/ProfilesEndpoints.cs b/Source/Reef/Api/ProfilesEndpoints.cs index 34363b5..5a782f2 100644 --- a/Source/Reef/Api/ProfilesEndpoints.cs +++ b/Source/Reef/Api/ProfilesEndpoints.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Mvc; using Reef.Core.Models; +using Reef.Core.Scripting; using Reef.Core.Services; using Reef.Core.TemplateEngines; using Serilog; @@ -46,6 +47,9 @@ public static void Map(WebApplication app) group.MapPost("/{id:int}/test-query", TestQuery); group.MapPost("/{id:int}/test-query-delta", TestQueryDelta); group.MapPost("/test-query-preview", TestQueryPreview); + group.MapPost("/test-script", TestScript); + group.MapGet("/interpreters", GetInterpreters); + group.MapPost("/interpreters/{name}/test", TestInterpreter); // Unified import profile execution endpoints (use ?type=import) group.MapPost("/{id:int}/execute", ExecuteProfile); @@ -1695,6 +1699,97 @@ private static async Task TestQueryPreview( } } + /// + /// Runs a Script-type pre/post-process step in isolation, without executing + /// the rest of the profile pipeline. Lets users iterate on a script + /// directly from the editor instead of having to run a full export to see + /// whether it works. + /// + private static async Task TestScript( + [FromBody] TestScriptRequest request, + IScriptRunner scriptRunner) + { + if (string.IsNullOrWhiteSpace(request.Interpreter)) + { + return Results.BadRequest(new { error = "Interpreter is required" }); + } + + if (string.IsNullOrWhiteSpace(request.ScriptPathOrInline)) + { + return Results.BadRequest(new { error = "Script is required" }); + } + + // Sample stdin payload mirroring what a real pre/post-processing run sends, + // so a script that reads context fields behaves the same way under test. + var sampleContext = System.Text.Json.JsonSerializer.Serialize(new + { + ExecutionId = 0, + ProfileId = request.ProfileId ?? 0, + RowCount = 10, + OutputPath = "/data/exports/sample-output.csv", + FileSizeBytes = 1024, + ExecutionTimeMs = 100, + OutputFormat = "CSV", + TriggeredBy = "Test", + StartedAt = DateTime.UtcNow, + CompletedAt = DateTime.UtcNow, + Status = "Success", + ErrorMessage = (string?)null, + parameters = new Dictionary() + }); + + try + { + var result = await scriptRunner.RunAsync(new ScriptExecutionRequest + { + Interpreter = request.Interpreter, + ScriptPathOrInline = request.ScriptPathOrInline, + IsInline = request.ScriptIsInline, + StdinJson = sampleContext, + TimeoutSeconds = request.Timeout ?? 30, + EnvAllowlist = request.EnvAllowlist + }); + + return Results.Ok(new + { + success = result.Success, + exitCode = result.ExitCode, + stdout = result.Stdout, + stderr = result.Stderr, + errorMessage = result.ErrorMessage, + elapsedMs = result.ElapsedMs + }); + } + catch (Exception ex) + { + Log.Error(ex, "Script test failed: {Message}", ex.Message); + return Results.Ok(new { success = false, exitCode = -1, stdout = "", stderr = "", errorMessage = ex.Message, elapsedMs = 0L }); + } + } + + /// + /// Checks which script interpreters (pwsh, python, node, bash, sh, cmd) are + /// actually runnable on this host, by running a trivial probe script through + /// each one via the real IScriptRunner path. Used by Admin's Interpreters + /// section and by the Profile editor to grey out Script interpreters that + /// won't work. Not Admin-gated - any authenticated user creating a profile + /// needs this to know which interpreters they can pick. + /// + private static async Task GetInterpreters(InterpreterService interpreterService) + { + var statuses = await interpreterService.CheckAllAsync(); + return Results.Ok(statuses); + } + + /// + /// Re-checks a single interpreter on demand (Admin's "Test" button per row). + /// + private static async Task TestInterpreter(string name, InterpreterService interpreterService) + { + var status = await interpreterService.CheckOneAsync(name); + return Results.Ok(status); + } + /// /// Request model for testing query preview without saved profile /// @@ -1704,6 +1799,19 @@ public class TestQueryPreviewRequest public string Query { get; set; } = string.Empty; } + /// + /// Request model for testing a Script-type pre/post-process step in isolation + /// + public class TestScriptRequest + { + public int? ProfileId { get; set; } + public string Interpreter { get; set; } = string.Empty; + public string ScriptPathOrInline { get; set; } = string.Empty; + public bool ScriptIsInline { get; set; } + public int? Timeout { get; set; } + public List? EnvAllowlist { get; set; } + } + /// /// Request model for testing query delta with optional custom query /// diff --git a/Source/Reef/Api/RecipesEndpoints.cs b/Source/Reef/Api/RecipesEndpoints.cs new file mode 100644 index 0000000..e8f02ee --- /dev/null +++ b/Source/Reef/Api/RecipesEndpoints.cs @@ -0,0 +1,213 @@ +using Microsoft.AspNetCore.Mvc; +using Reef.Core.Services; +using Serilog; + +namespace Reef.Api; + +public static class RecipesEndpoints +{ + public static void Map(WebApplication app) + { + var g = app.MapGroup("/api/recipes").RequireAuthorization(); + + g.MapGet("/", GetAvailable); + g.MapPost("/{key}/start", Start); + g.MapPost("/{key}/reconfigure", Reconfigure); + g.MapGet("/runs/{runId:int}", GetRunState); + g.MapPost("/runs/{runId:int}/steps/{stepKey}/save", SaveStep); + g.MapPost("/runs/{runId:int}/steps/{stepKey}/skip", SkipStep); + g.MapPost("/runs/{runId:int}/flow-groups/{flowGroup}/skip", SkipFlowGroup); + g.MapPost("/runs/{runId:int}/steps/{stepKey}/verify", VerifyStep); + g.MapPost("/runs/{runId:int}/complete", Complete); + g.MapPost("/runs/{runId:int}/abandon", Abandon); + g.MapGet("/runs/{runId:int}/reusable-destination", GetReusableDestination); + g.MapPost("/runs/{runId:int}/steps/{stepKey}/webhook", RegisterWebhook); + } + + private static async Task GetAvailable( + [FromServices] RecipeService svc, + HttpContext ctx) + { + try { return Results.Ok(await svc.GetAvailableRecipesAsync(GetUserId(ctx))); } + catch (Exception ex) { return ServerError("retrieving available recipes", ex); } + } + + private static async Task Start( + string key, + [FromServices] RecipeService svc, + HttpContext ctx, + CancellationToken ct) + { + try + { + var run = await svc.StartRecipeAsync(key, GetUserId(ctx), cloneFromRunId: null, ct); + var state = await svc.GetRunStateAsync(run.Id, ct); + return Results.Created($"/api/recipes/runs/{run.Id}", state); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"starting recipe '{key}'", ex); } + } + + private static async Task Reconfigure( + string key, + [FromBody] ReconfigureRequest request, + [FromServices] RecipeService svc, + HttpContext ctx, + CancellationToken ct) + { + try + { + var run = await svc.StartRecipeAsync(key, GetUserId(ctx), request.RunId, ct); + var state = await svc.GetRunStateAsync(run.Id, ct); + return Results.Created($"/api/recipes/runs/{run.Id}", state); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"reconfiguring recipe '{key}'", ex); } + } + + private static async Task GetRunState( + int runId, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try { return Results.Ok(await svc.GetRunStateAsync(runId, ct)); } + catch (InvalidOperationException ex) { return Results.NotFound(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"retrieving recipe run {runId}", ex); } + } + + private static async Task SaveStep( + int runId, + string stepKey, + [FromBody] Dictionary stepParams, + [FromServices] RecipeService svc, + HttpContext ctx, + CancellationToken ct) + { + try + { + var result = await svc.ExecuteStepAsync(runId, stepKey, stepParams, GetUserId(ctx), ct); + return Results.Ok(result); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"saving step '{stepKey}' for run {runId}", ex); } + } + + private static async Task SkipStep( + int runId, + string stepKey, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + var result = await svc.SkipStepAsync(runId, stepKey, ct); + return Results.Ok(result); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"skipping step '{stepKey}' for run {runId}", ex); } + } + + private static async Task SkipFlowGroup( + int runId, + string flowGroup, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + var result = await svc.SkipFlowGroupAsync(runId, Uri.UnescapeDataString(flowGroup), ct); + return Results.Ok(result); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"skipping flow group '{flowGroup}' for run {runId}", ex); } + } + + private static async Task VerifyStep( + int runId, + string stepKey, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + var result = await svc.VerifyStepAsync(runId, stepKey, ct); + return Results.Ok(result); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"verifying step '{stepKey}' for run {runId}", ex); } + } + + private static async Task Complete( + int runId, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + await svc.CompleteRunAsync(runId, ct); + return Results.Ok(new { message = "Recipe completed." }); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"completing recipe run {runId}", ex); } + } + + private static async Task Abandon( + int runId, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + await svc.AbandonRunAsync(runId, ct); + return Results.Ok(new { message = "Recipe abandoned." }); + } + catch (Exception ex) { return ServerError($"abandoning recipe run {runId}", ex); } + } + + private static async Task GetReusableDestination( + int runId, + [FromServices] RecipeService svc, + CancellationToken ct) + { + try + { + var destinationId = await svc.GetReusableDestinationIdAsync(runId, ct); + return Results.Ok(new { destinationId }); + } + catch (InvalidOperationException ex) { return Results.NotFound(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"retrieving reusable destination for run {runId}", ex); } + } + + private static async Task RegisterWebhook( + int runId, + string stepKey, + [FromServices] RecipeService svc, + HttpContext ctx, + CancellationToken ct) + { + try + { + var (webhookId, url) = await svc.RegisterTrackingWebhookAsync(runId, ctx.Request.Scheme, ctx.Request.Host.ToString(), ct); + return Results.Ok(new { webhookId, url }); + } + catch (InvalidOperationException ex) { return Results.BadRequest(new { error = ex.Message }); } + catch (Exception ex) { return ServerError($"registering Tracking Link webhook for run {runId}", ex); } + } + + private static int? GetUserId(HttpContext ctx) + { + var claim = ctx.User?.FindFirst("sub") ?? ctx.User?.FindFirst("userId"); + return claim is not null && int.TryParse(claim.Value, out var id) ? id : null; + } + + private static IResult ServerError(string context, Exception ex) + { + Log.Error(ex, "RecipesEndpoints: error {Context}", context); + return Results.Problem($"Error {context}: {ex.Message}"); + } + + public class ReconfigureRequest + { + public required int RunId { get; set; } + } +} diff --git a/Source/Reef/Core/Database/ScriptingMigration.cs b/Source/Reef/Core/Database/ScriptingMigration.cs new file mode 100644 index 0000000..eb2f01d --- /dev/null +++ b/Source/Reef/Core/Database/ScriptingMigration.cs @@ -0,0 +1,90 @@ +using Microsoft.Data.Sqlite; +using Dapper; +using Serilog; + +namespace Reef.Core.Database; + +/// +/// Database migration adding stdout/stderr/exit-code tracking columns to +/// ProfileExecutions, used by the Script pre/post-processing type so the +/// Execution History UI can surface what a script actually printed. +/// Run this on application startup. +/// +public class ScriptingMigration +{ + private readonly string _connectionString; + + public ScriptingMigration(string connectionString) + { + _connectionString = connectionString; + } + + public async Task ApplyAsync() + { + Log.Debug("Applying Scripting database migrations..."); + + try + { + using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(); + + await AddProfileExecutionColumnsAsync(conn); + + Log.Debug("✓ Scripting migrations completed"); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to apply Scripting migrations: {ErrorMessage}", ex.Message); + throw; + } + } + + private async Task AddProfileExecutionColumnsAsync(SqliteConnection conn) + { + var existingColumns = (await conn.QueryAsync( + @"SELECT name FROM pragma_table_info('ProfileExecutions') + WHERE name IN ( + 'PreProcessStdout', 'PreProcessStderr', 'PreProcessExitCode', + 'PostProcessStdout', 'PostProcessStderr', 'PostProcessExitCode' + )")).ToList(); + + if (!existingColumns.Contains("PreProcessStdout")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PreProcessStdout TEXT NULL"); + + if (!existingColumns.Contains("PreProcessStderr")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PreProcessStderr TEXT NULL"); + + if (!existingColumns.Contains("PreProcessExitCode")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PreProcessExitCode INTEGER NULL"); + + if (!existingColumns.Contains("PostProcessStdout")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PostProcessStdout TEXT NULL"); + + if (!existingColumns.Contains("PostProcessStderr")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PostProcessStderr TEXT NULL"); + + if (!existingColumns.Contains("PostProcessExitCode")) + await conn.ExecuteAsync("ALTER TABLE ProfileExecutions ADD COLUMN PostProcessExitCode INTEGER NULL"); + + Log.Debug("✓ ProfileExecutions scripting columns migration completed"); + } + + public async Task GetStatsAsync() + { + using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(); + + var columns = (await conn.QueryAsync( + @"SELECT name FROM pragma_table_info('ProfileExecutions') + WHERE name IN ( + 'PreProcessStdout', 'PreProcessStderr', 'PreProcessExitCode', + 'PostProcessStdout', 'PostProcessStderr', 'PostProcessExitCode' + )")).ToList(); + + return new + { + ColumnsAdded = columns.Count, + MigrationComplete = columns.Count == 6 + }; + } +} diff --git a/Source/Reef/Core/Models/ImportModels.cs b/Source/Reef/Core/Models/ImportModels.cs index 0ac39a7..4b9005f 100644 --- a/Source/Reef/Core/Models/ImportModels.cs +++ b/Source/Reef/Core/Models/ImportModels.cs @@ -208,6 +208,14 @@ public class ImportProfileExecution public string? ErrorMessage { get; set; } public string? StackTrace { get; set; } public string? ExecutionLog { get; set; } + + // Pre/post-process Script step observability (Type=Script only) + public string? PreProcessStdout { get; set; } + public string? PreProcessStderr { get; set; } + public int? PreProcessExitCode { get; set; } + public string? PostProcessStdout { get; set; } + public string? PostProcessStderr { get; set; } + public int? PostProcessExitCode { get; set; } public int FilesProcessed { get; set; } public long BytesProcessed { get; set; } diff --git a/Source/Reef/Core/Models/Models.cs b/Source/Reef/Core/Models/Models.cs index f3d6693..ccb126f 100644 --- a/Source/Reef/Core/Models/Models.cs +++ b/Source/Reef/Core/Models/Models.cs @@ -197,13 +197,19 @@ public class ProfileExecution public string? PreProcessStatus { get; set; } // null, Success, Failed, Skipped public string? PreProcessError { get; set; } public long? PreProcessTimeMs { get; set; } - + public string? PreProcessStdout { get; set; } // Script type only + public string? PreProcessStderr { get; set; } // Script type only + public int? PreProcessExitCode { get; set; } // Script type only + // Post-Processing Phase Tracking public DateTime? PostProcessStartedAt { get; set; } public DateTime? PostProcessCompletedAt { get; set; } public string? PostProcessStatus { get; set; } // null, Success, Failed, Skipped public string? PostProcessError { get; set; } public long? PostProcessTimeMs { get; set; } + public string? PostProcessStdout { get; set; } // Script type only + public string? PostProcessStderr { get; set; } // Script type only + public int? PostProcessExitCode { get; set; } // Script type only // Email Approval Workflow Tracking public string? ApprovalStatus { get; set; } // null, Pending, Approved, Rejected, Sent (tracks approval status separately) @@ -336,31 +342,55 @@ public class ScheduledTask public class ProcessingConfig { /// - /// Type of processing: Query or StoredProcedure + /// Type of processing: Query, StoredProcedure, or Script /// - public required string Type { get; set; } // Query, StoredProcedure - + public required string Type { get; set; } // Query, StoredProcedure, Script + /// /// SQL command to execute (can contain {placeholder} variables) /// For Query: SELECT, UPDATE, DELETE, INSERT, etc. /// For StoredProcedure: Procedure name (without EXEC/CALL) + /// Not used when Type is Script. /// - public required string Command { get; set; } - + public string? Command { get; set; } + /// /// Optional parameters for stored procedure or parameterized query /// public List? Parameters { get; set; } - + /// /// Timeout in seconds (default 30) /// public int Timeout { get; set; } = 30; - + /// /// Whether to continue execution if this processing step fails /// public bool ContinueOnError { get; set; } = false; + + /// + /// Script interpreter to use when Type is Script: pwsh, python, node, bash, sh, cmd + /// + public string? Interpreter { get; set; } + + /// + /// When Type is Script: either the inline script source or a path to a script + /// file, depending on . + /// + public string? ScriptPathOrInline { get; set; } + + /// + /// When Type is Script: whether ScriptPathOrInline holds inline source (true) + /// or a file path (false). + /// + public bool ScriptIsInline { get; set; } + + /// + /// When Type is Script: names of environment variables to pass through to + /// the script process. Everything else is withheld to avoid leaking secrets. + /// + public List? EnvAllowlist { get; set; } } /// @@ -481,6 +511,13 @@ public class ProcessingContext /// Available as: {splitkey} /// public string? SplitKey { get; set; } + + /// + /// Webhook/job trigger parameters passed into ExecuteProfileAsync. + /// Forwarded to Script-type processing steps as part of the stdin JSON + /// context, so a post-process webhook script can see what triggered the run. + /// + public Dictionary? WebhookParameters { get; set; } } // ===== Email Approval Workflow Models ===== diff --git a/Source/Reef/Core/Recipes/ErrorDigestEmailTemplate.cs b/Source/Reef/Core/Recipes/ErrorDigestEmailTemplate.cs new file mode 100644 index 0000000..bacd735 --- /dev/null +++ b/Source/Reef/Core/Recipes/ErrorDigestEmailTemplate.cs @@ -0,0 +1,150 @@ +// AUTO-CONSTRUCTED constant - DO NOT hand-edit the body below. +// Source: /Templates/Email_-_System_Error_Daily_Digest.ScribanTemplate.html (reference copy retained on disk; this is the seed copied into a QueryTemplates row at step-execution time) +namespace Reef.Core.Recipes; + +internal static class ErrorDigestEmailTemplate +{ + public const string DailyDigestHtml = """"""""""""""" +{{~ for row in rows ~}} + + + + + + {{~ row.subject | html.escape ~}} + + + + + + +{{~ end ~}} +"""""""""""""""; +} diff --git a/Source/Reef/Core/Recipes/ErrorDigestRecipe.cs b/Source/Reef/Core/Recipes/ErrorDigestRecipe.cs new file mode 100644 index 0000000..02c9d6e --- /dev/null +++ b/Source/Reef/Core/Recipes/ErrorDigestRecipe.cs @@ -0,0 +1,101 @@ +namespace Reef.Core.Recipes; + +// Second recipe proving the engine is generic: no staging table, no HTTP import, no +// webhook - just Connection/Destination/QueryTemplate/Profile steps. Required zero +// engine changes to add. +public static class ErrorDigestRecipe +{ + public const string Key = "system-error-daily-digest"; + + // No wizard-created staging table here - this assumes the user already has an + // error/log table, so the query is a starting point to edit, not a fixed schema. + public const string DefaultExportQuery = """ + SELECT + 'ops-team@yourcompany.com' AS email, + 'Daily System Report' AS subject, + 'RPT-' || strftime('%Y%m%d', 'now') AS report_id, + 'Your Company' AS company, + strftime('%Y-%m-%d %H:%M', 'now') AS generated_at, + 0 AS total_count, + 0 AS critical_count, + 0 AS warn_count, + 'https://dashboard.example.com/daily' AS dashboard_url, + 'https://example.com/unsubscribe' AS unsubscribe_url, + strftime('%Y', 'now') AS year, + '[]' AS error_list_json + """; + + public static Dictionary MockDigestRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["email"] = "ops-team@yourcompany.com", + ["subject"] = "Daily System Report: 2 Critical Events", + ["report_id"] = "RPT-20260101", + ["company"] = "Reef Demo Co", + ["generated_at"] = DateTime.UtcNow.ToString("MMM dd, yyyy HH:mm"), + ["total_count"] = "12", + ["critical_count"] = "2", + ["warn_count"] = "4", + ["dashboard_url"] = "https://dashboard.example.com/daily", + ["unsubscribe_url"] = "https://example.com/unsubscribe", + ["year"] = DateTime.UtcNow.Year.ToString(), + ["error_list_json"] = "[{\"time\":\"03:14:07\",\"level\":\"CRITICAL\",\"source\":\"PaymentGateway\",\"message\":\"Connection timed out waiting for response\",\"id\":\"ERR-4821\"}]" + }; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "System Error Daily Digest", + Description = "Email your team a daily summary of system errors/warnings pulled from your own database - no import required.", + Category = "Database Sync", + Icon = "alert-triangle", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null + }, + new() + { + StepKey = "destination", + Title = "Email Destination", + EntityType = RecipeEntityType.Destination, + IsShared = true, + VerifierKind = RecipeVerifierKind.EmailDestination + }, + new() + { + StepKey = "query-template", + Title = "Digest Email Template", + EntityType = RecipeEntityType.QueryTemplate, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "export-profile", + Title = "Send Daily Digest", + EntityType = RecipeEntityType.Profile, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Recipes/ExactGlobeRecipe.cs b/Source/Reef/Core/Recipes/ExactGlobeRecipe.cs new file mode 100644 index 0000000..f05770f --- /dev/null +++ b/Source/Reef/Core/Recipes/ExactGlobeRecipe.cs @@ -0,0 +1,251 @@ +namespace Reef.Core.Recipes; + +// Debtors Export (Flow A) + Items Export (Flow B) as one RecipeDefinition, both flows +// sharing Connection/Group (IsShared=true). Unlike WooCommerce/Magento, the Connection here +// is the user's own Exact Globe+ business database - a real, manual pick, never auto-provisioned +// staging. There's no import side (queries the user's existing tables directly, same shape as +// ErrorDigestRecipe) and no email Destination step - this is a file export to XML, using +// Profile's inline OutputDestinationType="Local"/OutputDestinationConfig path so no separate +// Destination entity needs to be created first. +public static class ExactGlobeRecipe +{ + public const string Key = "exact-globe-data-export"; + + public const string DebtorsFlowGroup = "Debtors Export"; + public const string ItemsFlowGroup = "Items Export"; + + // Trimmed from the full example in Exact_Globe_-_eExact_XML_for_Debtors.ScribanTemplate.md - + // same column aliases the template expects, with the OUTER APPLY/VAT join dropped since + // they're optional enrichments, not required for a working starting query. + public const string DefaultDebtorsExportQuery = """ + SELECT + c.debcode AS EntityCode, + c.cmp_code AS Code, + c.cmp_status AS Status, + c.cmp_type AS Type, + TRIM(c.cmp_name) AS Name, + COALESCE(c.TextField1, '') AS FreeText1, + COALESCE(c.TextField2, '') AS FreeText2, + COALESCE(c.TextField3, '') AS FreeText3, + COALESCE(c.TextField4, '') AS FreeText4, + COALESCE(c.TextField5, '') AS FreeText5, + COALESCE(c.TextField6, '') AS FreeText6, + COALESCE(c.TextField7, '') AS FreeText7, + COALESCE(c.TextField8, '') AS FreeText8, + COALESCE(c.TextField9, '') AS FreeText9, + COALESCE(c.TextField10, '') AS FreeText10, + COALESCE(c.NumberField1, 0) AS FreeNumber1, + COALESCE(c.NumberField2, 0) AS FreeNumber2, + COALESCE(c.NumberField3, 0) AS FreeNumber3, + COALESCE(c.NumberField4, 0) AS FreeNumber4, + COALESCE(c.NumberField5, 0) AS FreeNumber5, + COALESCE(c.YesNoField1, 0) AS FreeYesNo1, + COALESCE(c.YesNoField2, 0) AS FreeYesNo2, + COALESCE(c.YesNoField3, 0) AS FreeYesNo3, + COALESCE(c.YesNoField4, 0) AS FreeYesNo4, + COALESCE(c.YesNoField5, 0) AS FreeYesNo5, + FORMAT(c.syscreated, 'yyyy-MM-dd HH:mm') AS CreatedDate, + FORMAT(c.sysmodified, 'yyyy-MM-dd HH:mm') AS ModifiedDate, + REPLACE(REPLACE(CAST(c.sysguid AS VARCHAR(36)), '{', ''), '}', '') AS UUID + FROM dbo.cicmpy c + WHERE c.debcode IS NOT NULL + ORDER BY c.debcode + """; + + // Trimmed from Exact_Globe_-_eExact_XML_for_Items.ScribanTemplate.md - drops the + // supplier/warehouse JSON subqueries and item-class joins (optional enrichments the + // template already guards with null checks) but keeps every column the template expects + // a value for so the default query renders cleanly out of the box. + public const string DefaultItemsExportQuery = """ + SELECT + i.ItemCode AS ItemCode, + i.Type AS Type, + TRIM(i.SearchCode) AS SearchCode, + i.Description AS Description, + COALESCE(i.Description_0, '') AS Description_0, + COALESCE(i.Description_1, '') AS Description_1, + COALESCE(i.Description_2, '') AS Description_2, + COALESCE(i.Description_3, '') AS Description_3, + COALESCE(i.Description_4, '') AS Description_4, + i.Condition AS Condition, + i.IsSalesItem AS IsSalesItem, + i.IsPurchaseItem AS IsPurchaseItem, + i.IsSerialNumberItem AS IsSerialNumberItem, + i.IsBatchItem AS IsBatchItem, + i.IsSubAssemblyItem AS IsSubAssemblyItem, + i.IsAssembled AS IsAssembledItem, + i.IsStockItem AS IsStockItem, + i.IsBackOrderItem AS IsBackOrderItem, + i.IsFractionAllowedItem AS IsFractionAllowedItem, + i.IsPriceRegulationItem AS IsPriceRegulationItem, + i.IsTextItem AS IsTextItem, + i.IsDiscount AS IsDiscountItem, + i.IsExplode AS IsExplodeItem, + i.IsPrintItem AS IsPrintItem, + i.IsOutsourcedItem AS IsOutsourcedItem, + i.IsServiceItem AS IsServiceItem, + i.SalesPackagePrice AS SalesPrice, + i.CostPriceCurrency AS SalesCurrency, + TRIM(i.PackageDescription) AS SalesUnit, + i.CostPriceCurrency AS CostCurrency, + i.CostPriceStandard AS CostPrice, + i.PurchaseCurrency AS PurchaseCurrency, + i.PurchasePrice AS PurchasePrice, + i.NetWeight AS NetWeight, + i.GrossWeight AS GrossWeight, + COALESCE(i.UserField_01, '') AS FreeText1, + COALESCE(i.UserField_02, '') AS FreeText2, + COALESCE(i.UserField_03, '') AS FreeText3, + COALESCE(i.UserField_04, '') AS FreeText4, + COALESCE(i.UserField_05, '') AS FreeText5, + COALESCE(i.UserNumber_01, 0) AS FreeNumber1, + COALESCE(i.UserNumber_02, 0) AS FreeNumber2, + COALESCE(i.UserNumber_03, 0) AS FreeNumber3, + COALESCE(i.UserNumber_04, 0) AS FreeNumber4, + COALESCE(i.UserNumber_05, 0) AS FreeNumber5, + COALESCE(i.UserYesNo_01, 0) AS FreeYesNo1, + COALESCE(i.UserYesNo_02, 0) AS FreeYesNo2, + COALESCE(i.UserYesNo_03, 0) AS FreeYesNo3, + COALESCE(i.UserYesNo_04, 0) AS FreeYesNo4, + COALESCE(i.UserYesNo_05, 0) AS FreeYesNo5, + i.ShelfLife AS ShelfLife, + i.Warranty AS Warranty, + i.IntrastatEnabled AS IntrastatEnabled, + i.AddExtraReceiptToOrder AS AddExtraReceiptToOrder, + i.IsCommissionable AS IsCommissionable, + TRIM(i.CommissionMethod) AS CommissionMethod, + i.CommissionValue AS CommissionValue, + i.TaxItemClassification AS TaxItemClassification, + FORMAT(i.syscreated, 'yyyy-MM-dd HH:mm') AS CreatedDate, + FORMAT(i.sysmodified, 'yyyy-MM-dd HH:mm') AS ModifiedDate, + REPLACE(REPLACE(CAST(i.sysguid AS VARCHAR(36)), '{', ''), '}', '') AS UUID + FROM dbo.Items i + WHERE i.IsSalesItem = 1 + AND i.Type NOT IN ('L', 'P') + ORDER BY i.ItemCode + """; + + // Mock rows for the ScribanTemplateVerifier when the staging/query table can't be probed + // for a real row (this recipe has no staging table at all) - shaped to match every field + // the two XML templates reference so a preview render always succeeds. + public static Dictionary MockDebtorRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["Code"] = "DEB001", + ["Status"] = "Active", + ["Type"] = "C", + ["Name"] = "Reef Demo Customer BV", + ["FreeText1"] = "", ["FreeText2"] = "", ["FreeText3"] = "", ["FreeText4"] = "", ["FreeText5"] = "", + ["FreeText6"] = "", ["FreeText7"] = "", ["FreeText8"] = "", ["FreeText9"] = "", ["FreeText10"] = "", + ["FreeNumber1"] = 0, ["FreeNumber2"] = 0, ["FreeNumber3"] = 0, ["FreeNumber4"] = 0, ["FreeNumber5"] = 0, + ["FreeYesNo1"] = 0, ["FreeYesNo2"] = 0, ["FreeYesNo3"] = 0, ["FreeYesNo4"] = 0, ["FreeYesNo5"] = 0, + ["CreatedDate"] = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm"), + ["ModifiedDate"] = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm"), + ["UUID"] = Guid.NewGuid().ToString() + }; + + public static Dictionary MockItemRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["ItemCode"] = "ITEM001", + ["Type"] = "V", + ["SearchCode"] = "ITEM001", + ["Description"] = "Reef Demo Item", + ["Description_0"] = "", ["Description_1"] = "", ["Description_2"] = "", ["Description_3"] = "", ["Description_4"] = "", + ["IsSalesItem"] = true, ["IsPurchaseItem"] = true, ["IsSerialNumberItem"] = false, ["IsBatchItem"] = false, + ["IsSubAssemblyItem"] = false, ["IsAssembledItem"] = false, ["IsStockItem"] = true, ["IsBackOrderItem"] = false, + ["IsFractionAllowedItem"] = false, ["IsPriceRegulationItem"] = false, ["IsTextItem"] = false, + ["IsDiscountItem"] = false, ["IsExplodeItem"] = false, ["IsPrintItem"] = true, ["IsOutsourcedItem"] = false, + ["IsServiceItem"] = false, + ["SalesPrice"] = 19.99m, ["SalesCurrency"] = "EUR", ["SalesUnit"] = "PCS", + ["CostPrice"] = 9.99m, ["CostCurrency"] = "EUR", + ["PurchasePrice"] = 9.99m, ["PurchaseCurrency"] = "EUR", + ["FreeText1"] = "", ["FreeText2"] = "", ["FreeText3"] = "", ["FreeText4"] = "", ["FreeText5"] = "", + ["FreeNumber1"] = 0, ["FreeNumber2"] = 0, ["FreeNumber3"] = 0, ["FreeNumber4"] = 0, ["FreeNumber5"] = 0, + ["FreeYesNo1"] = 0, ["FreeYesNo2"] = 0, ["FreeYesNo3"] = 0, ["FreeYesNo4"] = 0, ["FreeYesNo5"] = 0, + ["CreatedDate"] = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm"), + ["ModifiedDate"] = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm"), + ["UUID"] = Guid.NewGuid().ToString() + }; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "Exact Globe+ Data Export", + Description = "Export Debtors (customers) or Items (products) from your Exact Globe+ database to eExact-compliant XML files - no import required, queries your own tables directly.", + Category = "ERP", + Icon = "database", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection + // Deliberately no CanAutoProvision: this Connection is the user's real Exact + // Globe+ business database, never a Reef-managed staging file. + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null, + CanAutoProvision = true + }, + + new() + { + StepKey = "debtors-query-template", + Title = "Debtors XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = DebtorsFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "debtors-export-profile", + Title = "Export Debtors to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = DebtorsFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "debtors-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = DebtorsFlowGroup, + VerifierKind = null + }, + + new() + { + StepKey = "items-query-template", + Title = "Items XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = ItemsFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "items-export-profile", + Title = "Export Items to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = ItemsFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "items-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = ItemsFlowGroup, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Recipes/ExactGlobeTemplates.cs b/Source/Reef/Core/Recipes/ExactGlobeTemplates.cs new file mode 100644 index 0000000..089a442 --- /dev/null +++ b/Source/Reef/Core/Recipes/ExactGlobeTemplates.cs @@ -0,0 +1,293 @@ +// AUTO-CONSTRUCTED constant - DO NOT hand-edit the body below. +// Source: /Templates/Exact_Globe_-_eExact_XML_for_Debtors.ScribanTemplate.xml and +// /Templates/Exact_Globe_-_eExact_XML_for_Items.ScribanTemplate.xml (reference copies retained +// on disk; these are the seed copies copied into QueryTemplates rows at step-execution time). +namespace Reef.Core.Recipes; + +internal static class ExactGlobeTemplates +{ + public const string DebtorsXmlTemplate = """"""""""""""" + + + + {{ date.now | date.to_string '%Y-%m-%d.%H:%M' }} + + +{{~ for row in rows ~}} + + + {{ row.Name }} + + + + + {{ row.FreeText1 }} + {{ row.FreeText2 }} + {{ row.FreeText3 }} + {{ row.FreeText4 }} + {{ row.FreeText5 }} + {{ row.FreeText6 }} + {{ row.FreeText7 }} + {{ row.FreeText8 }} + {{ row.FreeText9 }} + {{ row.FreeText10 }} + + + + + {{ row.FreeNumber1 | math.format '0.00000' }} + {{ row.FreeNumber2 | math.format '0.00000' }} + {{ row.FreeNumber3 | math.format '0.00000' }} + {{ row.FreeNumber4 | math.format '0.00000' }} + {{ row.FreeNumber5 | math.format '0.00000' }} + + + + {{ row.FreeYesNo1 }} + {{ row.FreeYesNo2 }} + {{ row.FreeYesNo3 }} + {{ row.FreeYesNo4 }} + {{ row.FreeYesNo5 }} + + + + + {{ row.CreatedDate }} + {{ row.ModifiedDate }} + {{ row.UUID }} + +{{~ end ~}} + + +"""""""""""""""; + + public const string ItemsXmlTemplate = """"""""""""""" + + + {{ date.now | date.to_string '%Y-%m-%d.%H:%M' }} + + +{{~ for row in rows ~}} + + {{ row.Description }} + + + {{~ for i in 0..4 ~}} + {{~ desc_field = "Description_" + i }} + {{~ desc_value = row[desc_field] }} + {{~ if desc_value != "" && desc_value != null ~}} + {{ desc_value }} + {{~ end ~}} + {{~ end ~}} + + + {{~ if row.AssortmentNumber != null ~}} + + {{ row.AssortmentDescription }} + + {{~ end ~}} + + {{~ if row.AvailableFrom != null && row.AvailableFrom != "" ~}} + + {{ row.AvailableFrom }} + + {{~ end ~}} + + {{~ if row.Condition != null ~}} + {{ row.Condition }} + {{~ end ~}} + + {{ row.IsSalesItem }} + {{ row.IsPurchaseItem }} + {{ row.IsSerialNumberItem }} + {{ row.IsBatchItem }} + {{ row.IsSubAssemblyItem }} + {{ row.IsAssembledItem }} + {{ row.IsStockItem }} + {{ row.IsBackOrderItem }} + {{ row.IsFractionAllowedItem }} + {{ row.IsPriceRegulationItem }} + {{ row.IsTextItem }} + {{ row.IsDiscountItem }} + {{ row.IsExplodeItem }} + {{ row.IsPrintItem }} + {{ row.IsOutsourcedItem }} + + + + + {{ (row.SalesPrice | default 0) | math.format '0.000' }} + {{~ if row.SalesVatCode != null && row.SalesVatCode != "" ~}} + + {{ row.SalesVatDescription }} + {{ row.SalesVatPercentage }} + + {{~ end ~}} + + + + + + + + {{ (row.CostPrice | default 0) | math.format '0.00000' }} + + + + {{~ if row.NetWeight != null || row.GrossWeight != null ~}} + + {{~ if row.NetWeight != null ~}} + {{ row.NetWeight | math.format '0.00000' }} + {{~ end ~}} + {{~ if row.GrossWeight != null ~}} + {{ row.GrossWeight | math.format '0.00000' }} + {{~ end ~}} + + {{~ end ~}} + + + + {{~ for i in 1..5 ~}} + {{~ field_name = "FreeText" + i }} + {{~ field_value = row[field_name] }} + {{~ if field_value != "" && field_value != null ~}} + {{ field_value }} + {{~ end ~}} + {{~ end ~}} + + + + {{~ for i in 1..5 ~}} + {{~ field_name = "FreeDate" + i }} + {{~ field_value = row[field_name] }} + {{~ if field_value != "" && field_value != null ~}} + {{ field_value }} + {{~ end ~}} + {{~ end ~}} + + + + {{~ for i in 1..5 ~}} + {{~ field_name = "FreeNumber" + i }} + {{~ field_value = row[field_name] }} + {{~ if field_value != "" && field_value != null ~}} + {{ field_value | math.format '0.00000' }} + {{~ end ~}} + {{~ end ~}} + + + + {{~ for i in 1..5 ~}} + {{~ field_name = "FreeYesNo" + i }} + {{~ field_value = row[field_name] }} + {{~ if field_value != "" && field_value != null ~}} + {{ field_value }} + {{~ end ~}} + {{~ end ~}} + + + + {{~ if row.Class1_Code != null && row.Class1_Code != "" ~}} + + {{ row.Class1_Description }} + + {{~ end ~}} + {{~ if row.Class2_Code != null && row.Class2_Code != "" ~}} + + {{ row.Class2_Description }} + + {{~ end ~}} + {{~ if row.Class3_Code != null && row.Class3_Code != "" ~}} + + {{ row.Class3_Description }} + + {{~ end ~}} + {{~ if row.Class4_Code != null && row.Class4_Code != "" ~}} + + {{ row.Class4_Description }} + + {{~ end ~}} + {{~ if row.Class5_Code != null && row.Class5_Code != "" ~}} + + {{ row.Class5_Description }} + + {{~ end ~}} + + {{~ if row.SuppliersJSON != null ~}} + {{~ suppliers = row.SuppliersJSON | parse_json }} + {{~ for supplier in suppliers ~}} + + + + {{ supplier.AccountName }} + + {{ row.ItemCode }} + {{ supplier.SupplierPreference }} + {{~ if supplier.EANCode != null && supplier.EANCode != "" ~}} + {{ supplier.EANCode }} + {{~ end ~}} + + + + {{ (row.PurchasePrice | default 0) | math.format '0.000' }} + + + + {{ supplier.DeliveryTimeInDays }} + {{ supplier.DeliverableFromStock }} + {{ supplier.DropShip }} + + + + + {{~ end ~}} + {{~ end ~}} + + {{~ if row.WarehousesJSON != null ~}} + {{~ warehouses = row.WarehousesJSON | parse_json }} + + {{~ for warehouse in warehouses ~}} + + + {{ warehouse.Description }} + + + {{~ end ~}} + + {{~ end ~}} + + {{~ if row.ShelfLife != null ~}} + {{ row.ShelfLife }} + {{~ end ~}} + {{~ if row.Warranty != null ~}} + {{ row.Warranty }} + {{~ end ~}} + {{ row.IsServiceItem }} + {{~ if row.IntrastatEnabled != null ~}} + {{ row.IntrastatEnabled }} + {{~ end ~}} + {{~ if row.AddExtraReceiptToOrder != null ~}} + {{ row.AddExtraReceiptToOrder }} + {{~ end ~}} + {{~ if row.IsCommissionable != null ~}} + {{ row.IsCommissionable }} + {{~ end ~}} + {{~ if row.CommissionMethod != null && row.CommissionMethod != "" ~}} + {{ row.CommissionMethod }} + {{~ end ~}} + {{~ if row.CommissionValue != null ~}} + {{ row.CommissionValue | math.format '0.00000' }} + {{~ end ~}} + {{~ if row.TaxItemClassification != null ~}} + {{ row.TaxItemClassification }} + {{~ end ~}} + + {{ row.CreatedDate }} + {{ row.ModifiedDate }} + {{ row.UUID }} + +{{~ end ~}} + + +"""""""""""""""; +} diff --git a/Source/Reef/Core/Recipes/IRecipeVerifier.cs b/Source/Reef/Core/Recipes/IRecipeVerifier.cs new file mode 100644 index 0000000..f432d9f --- /dev/null +++ b/Source/Reef/Core/Recipes/IRecipeVerifier.cs @@ -0,0 +1,27 @@ +namespace Reef.Core.Recipes; + +public class RecipeVerifyResult +{ + public required bool Success { get; set; } + public required string Message { get; set; } + public string? Detail { get; set; } +} + +public class RecipeVerifyContext +{ + public int? EntityId { get; set; } + public int? ConnectionId { get; set; } + public string? TableName { get; set; } + public Dictionary Params { get; set; } = []; + + // Lets each recipe supply mock data shaped for its own template's columns instead of + // the verifier hardcoding one recipe's shape. Null falls back to the verifier's + // built-in WooCommerce-shaped default. + public Dictionary? MockTemplateRow { get; set; } +} + +// Real, live verification of a step's entity - never just "saved to DB". +public interface IRecipeVerifier +{ + Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default); +} diff --git a/Source/Reef/Core/Recipes/MagentoEmailTemplates.cs b/Source/Reef/Core/Recipes/MagentoEmailTemplates.cs new file mode 100644 index 0000000..b673a0d --- /dev/null +++ b/Source/Reef/Core/Recipes/MagentoEmailTemplates.cs @@ -0,0 +1,223 @@ +// AUTO-CONSTRUCTED constant - DO NOT hand-edit the body below. +// Source: /Templates/Magento_-_Email_Tracking_Update.ScribanTemplate.html (reference copy retained on disk; this is the seed copied into a QueryTemplates row at step-execution time) +namespace Reef.Core.Recipes; + +internal static class MagentoEmailTemplates +{ + public const string TrackingUpdateHtml = """"""""""""""" +{{~ for row in rows ~}} + + + + + + A shipment for your order {{~ row.order_number | html.escape ~}} is on its way + + + + + + +{{~ end ~}} +"""""""""""""""; +} diff --git a/Source/Reef/Core/Recipes/MagentoRecipe.cs b/Source/Reef/Core/Recipes/MagentoRecipe.cs new file mode 100644 index 0000000..2d13352 --- /dev/null +++ b/Source/Reef/Core/Recipes/MagentoRecipe.cs @@ -0,0 +1,169 @@ +namespace Reef.Core.Recipes; + +// Single-flow recipe (unlike WooCommerceRecipe's two flows) - pulls shipment tracking data +// from Magento's REST API and emails customers an update. Step keys are plain (no +// "shipments-" prefix) since there's only one flow here; RecipeService dispatches on +// recipe.Key instead, the same (recipeKey, isShipments) switch-tuple pattern already used +// for ErrorDigestRecipe vs WooCommerceRecipe. +public static class MagentoRecipe +{ + public const string Key = "magento-tracking-link"; + public const string StagingTableName = "MagentoShipments"; + + // Per-database-type identity/text syntax for Reef's supported query sources. + // Sqlite here is the user's chosen staging file (never Reef's own app database). + public static string GetStagingTableDdl(string connectionType) => connectionType switch + { + "SqlServer" => """ + CREATE TABLE MagentoShipments ( + Id INT IDENTITY(1,1) PRIMARY KEY, + MagentoOrderId BIGINT NOT NULL, + OrderNumber NVARCHAR(50) NULL, + CustomerName NVARCHAR(255) NULL, + CustomerEmail NVARCHAR(255) NULL, + Company NVARCHAR(255) NULL, + TrackingNumber NVARCHAR(100) NULL, + Carrier NVARCHAR(100) NULL, + TrackingUrl NVARCHAR(500) NULL, + EstimatedDeliveryDate NVARCHAR(50) NULL, + ShippingAddressJson NVARCHAR(MAX) NULL, + ItemsJson NVARCHAR(MAX) NULL, + EmailSent BIT NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT GETUTCDATE(), + CONSTRAINT UQ_MagentoShipments_MagentoOrderId UNIQUE (MagentoOrderId) + ) + """, + "MySQL" or "MariaDB" => """ + CREATE TABLE MagentoShipments ( + Id INT AUTO_INCREMENT PRIMARY KEY, + MagentoOrderId BIGINT NOT NULL, + OrderNumber VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + TrackingNumber VARCHAR(100) NULL, + Carrier VARCHAR(100) NULL, + TrackingUrl VARCHAR(500) NULL, + EstimatedDeliveryDate VARCHAR(50) NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent TINYINT(1) NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY UQ_MagentoShipments_MagentoOrderId (MagentoOrderId) + ) + """, + "Sqlite" => """ + CREATE TABLE MagentoShipments ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + MagentoOrderId INTEGER NOT NULL, + OrderNumber TEXT NULL, + CustomerName TEXT NULL, + CustomerEmail TEXT NULL, + Company TEXT NULL, + TrackingNumber TEXT NULL, + Carrier TEXT NULL, + TrackingUrl TEXT NULL, + EstimatedDeliveryDate TEXT NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent INTEGER NOT NULL DEFAULT 0, + CreatedAt TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (MagentoOrderId) + ) + """, + _ => """ + CREATE TABLE MagentoShipments ( + Id SERIAL PRIMARY KEY, + MagentoOrderId BIGINT NOT NULL UNIQUE, + OrderNumber VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + TrackingNumber VARCHAR(100) NULL, + Carrier VARCHAR(100) NULL, + TrackingUrl VARCHAR(500) NULL, + EstimatedDeliveryDate VARCHAR(50) NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP NOT NULL DEFAULT NOW() + ) + """ + }; + + public const string DefaultExportQuery = "SELECT * FROM MagentoShipments WHERE EmailSent = 0"; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "Magento Tracking Link", + Description = "Pull shipment tracking data from your Magento store's REST API and automatically email customers a tracking-link update.", + Category = "E-commerce", + Icon = "package", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection, + CanAutoProvision = true + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null, + CanAutoProvision = true + }, + new() + { + StepKey = "destination", + Title = "Email Destination", + EntityType = RecipeEntityType.Destination, + IsShared = true, + VerifierKind = RecipeVerifierKind.EmailDestination + }, + new() + { + StepKey = "staging-table", + Title = "Staging Table", + EntityType = RecipeEntityType.StagingTable, + VerifierKind = RecipeVerifierKind.StagingTable + }, + new() + { + StepKey = "import-profile", + Title = "Pull Shipments from Magento", + EntityType = RecipeEntityType.ImportProfile, + VerifierKind = RecipeVerifierKind.HttpSource + }, + new() + { + StepKey = "query-template", + Title = "Email Template", + EntityType = RecipeEntityType.QueryTemplate, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "export-profile", + Title = "Send Tracking Emails", + EntityType = RecipeEntityType.Profile, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Recipes/RecipeDefinition.cs b/Source/Reef/Core/Recipes/RecipeDefinition.cs new file mode 100644 index 0000000..8d72519 --- /dev/null +++ b/Source/Reef/Core/Recipes/RecipeDefinition.cs @@ -0,0 +1,63 @@ +namespace Reef.Core.Recipes; + +// Recipe definitions live in code, not the DB - only a handful ship and verification +// logic is bespoke per step, so a DB-driven recipe DSL would solve a problem that +// doesn't exist yet. Run progress (resumability) is what needs persistence - see +// RecipeRuns / RecipeService. +public class RecipeDefinition +{ + public required string Key { get; set; } + public required string Name { get; set; } + public required string Description { get; set; } + public required string Category { get; set; } + public required string Icon { get; set; } + public List Steps { get; set; } = new(); +} + +public class RecipeStepDefinition +{ + public required string StepKey { get; set; } + public required string Title { get; set; } + public required RecipeEntityType EntityType { get; set; } + public bool IsOptional { get; set; } = false; + + // Shared infrastructure reused across flows (Connection/Group/Destination); renders + // under a "Shared" heading instead of its flow's own section. + public bool IsShared { get; set; } = false; + + public RecipeVerifierKind? VerifierKind { get; set; } + + // UI grouping for non-shared steps (e.g. "Order Confirmation", "Tracking Link") so one + // RecipeDefinition can hold multiple flows without flattening them into one step list. + public string? FlowGroup { get; set; } + + // Opt-in for "simple mode": when true and the run has no saved value for this step yet, + // RecipeService.StartRecipeAsync silently provisions (find-or-create) a sensible default + // entity - a dedicated Sqlite staging Connection, or a Group named after the recipe - and + // seeds it into StepStateJson as Verified. The wizard's step rail hides these steps unless + // the user flips the "Advanced" toggle, letting them override with a real connection/group. + public bool CanAutoProvision { get; set; } = false; +} + +public enum RecipeEntityType +{ + Connection, + Group, + Destination, + StagingTable, + ImportProfile, + QueryTemplate, + Profile, + Job, + Webhook +} + +public enum RecipeVerifierKind +{ + Connection, + HttpSource, + StagingTable, + EmailDestination, + ScribanTemplate, + ExportQuery +} diff --git a/Source/Reef/Core/Recipes/RecipeRegistry.cs b/Source/Reef/Core/Recipes/RecipeRegistry.cs new file mode 100644 index 0000000..fc69edc --- /dev/null +++ b/Source/Reef/Core/Recipes/RecipeRegistry.cs @@ -0,0 +1,24 @@ +namespace Reef.Core.Recipes; + +/// +/// The extension point for Store recipes. Phase 1 shipped WooCommerce Order Confirmation; +/// Phase 3 added System Error Daily Digest (ErrorDigestRecipe) as a second, deliberately +/// simpler recipe to prove the engine is genuinely generic - it required zero changes to +/// RecipeService, RecipesEndpoints, or any IRecipeVerifier. Adding a future recipe means +/// adding another RecipeDefinition here, nothing else. +/// +public static class RecipeRegistry +{ + public static IReadOnlyList All { get; } = new List + { + WooCommerceRecipe.Definition, + WooCommerceTrackingRecipe.Definition, + ErrorDigestRecipe.Definition, + MagentoRecipe.Definition, + ExactGlobeRecipe.Definition, + UblExportRecipe.Definition + }; + + public static RecipeDefinition? GetByKey(string key) => + All.FirstOrDefault(r => r.Key.Equals(key, StringComparison.OrdinalIgnoreCase)); +} diff --git a/Source/Reef/Core/Recipes/RecipeRun.cs b/Source/Reef/Core/Recipes/RecipeRun.cs new file mode 100644 index 0000000..b35e5ac --- /dev/null +++ b/Source/Reef/Core/Recipes/RecipeRun.cs @@ -0,0 +1,47 @@ +namespace Reef.Core.Recipes; + +/// +/// Persisted progress for one user's walk through a recipe. Mirrors the RecipeRuns +/// table. Recipe *definitions* live in code (RecipeRegistry); only run *progress* +/// needs DB persistence, so a user can close the wizard mid-setup and resume later +/// with already-created/verified entities intact. +/// +public class RecipeRun +{ + public int Id { get; set; } + public required string RecipeKey { get; set; } + + /// InProgress, Completed, Abandoned. + public string Status { get; set; } = "InProgress"; + + public string? CurrentStepKey { get; set; } + + /// JSON: { stepKey: { entityId, verified, lastVerifiedAt, paramsJson } } + public string? StepStateJson { get; set; } + + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; + public int? CreatedBy { get; set; } + public DateTime? CompletedAt { get; set; } +} + +/// +/// Per-step state stored inside RecipeRun.StepStateJson, keyed by RecipeStepDefinition.StepKey. +/// +public class RecipeStepState +{ + /// Id of the entity this step created/updated (Connection, ImportProfile, Destination, etc.). + public int? EntityId { get; set; } + + public bool Verified { get; set; } + + public DateTime? LastVerifiedAt { get; set; } + + public string? LastVerifyMessage { get; set; } + + /// Raw params the step was last saved with - lets verifiers (e.g. ExportQueryVerifier) see step-specific config like the query text without re-fetching the entity. + public Dictionary Params { get; set; } = new(); + + /// True when the user explicitly chose to skip an optional step. + public bool Skipped { get; set; } +} diff --git a/Source/Reef/Core/Recipes/UblExportRecipe.cs b/Source/Reef/Core/Recipes/UblExportRecipe.cs new file mode 100644 index 0000000..6e11c95 --- /dev/null +++ b/Source/Reef/Core/Recipes/UblExportRecipe.cs @@ -0,0 +1,310 @@ +namespace Reef.Core.Recipes; + +// Four flows (Invoice/Order/Despatch Advice/Inventory) as one RecipeDefinition, sharing +// Connection/Group (IsShared=true) - same shape as ExactGlobeRecipe's Debtors/Items pair, +// just twice as many flows. Unlike ExactGlobeRecipe this isn't tied to one ERP vendor: the +// XML produced is plain UBL 2.1, so the Connection is still the user's own real business +// database (never auto-provisioned staging - same reasoning as ExactGlobe), and there's no +// import side and no email Destination step, just a file export using Profile's inline +// OutputDestinationType="Local"/OutputDestinationConfig path. +public static class UblExportRecipe +{ + public const string Key = "ubl-standard-export"; + + public const string InvoiceFlowGroup = "Invoice Export"; + public const string OrderFlowGroup = "Order Export"; + public const string DespatchFlowGroup = "Despatch Advice Export"; + public const string InventoryFlowGroup = "Inventory Export"; + + // Trimmed from the example in ERP_Example_-_Invoice_UBL_2.1_ScribanTemplate.md - the doc's + // SQL Server example builds the *_json columns with JSON_QUERY()/string literals; here + // they're built generically from the user's own tables with a vendor-neutral subquery + // shape (JSON building syntax differs across SQL Server/MySQL/PostgreSQL, so this is an + // illustrative starting point, not a copy-pasteable query for a specific database). + public const string DefaultInvoiceExportQuery = """ + SELECT + i.InvoiceNumber AS invoice_id, + i.SupplierName AS supplier_name, + i.CustomerName AS customer_name, + i.InvoiceTypeCode AS invoice_type_code, + '2.1' AS ubl_version, + i.CustomizationId AS customization_id, + i.IssueDate AS issue_date, + i.DueDate AS due_date, + i.Currency AS currency, + i.TotalNet AS total_net, + i.TotalTax AS total_tax, + i.TotalGross AS total_gross, + i.PaymentMeansCode AS payment_means_code, + i.PayeeIban AS payee_iban, + i.SupplierPartyJson AS supplier_party_json, + i.CustomerPartyJson AS customer_party_json, + i.TaxSummaryJson AS tax_summary_json, + i.LineItemsJson AS line_items_json + FROM Invoices i + WHERE i.Exported = 0 + ORDER BY i.InvoiceNumber + """; + + // Trimmed from ERP_Example_-_Order_UBL_2.1_ScribanTemplate.md, same generic-JSON-column + // approach as the invoice default above. + public const string DefaultOrderExportQuery = """ + SELECT + o.OrderNumber AS order_id, + o.IssueDate AS issue_date, + o.Currency AS currency, + o.Note AS note, + '2.1' AS ubl_version, + o.CustomizationId AS customization_id, + o.TotalNet AS total_net, + o.TotalTax AS total_tax, + o.TotalGross AS total_gross, + o.BuyerPartyJson AS buyer_party_json, + o.SellerPartyJson AS seller_party_json, + o.DeliveryJson AS delivery_json, + o.LineItemsJson AS line_items_json + FROM Orders o + WHERE o.Exported = 0 + ORDER BY o.OrderNumber + """; + + // Trimmed from ERP_Example_-_Despatch_Advice_UBL_2.1_ScribanTemplate.md. + public const string DefaultDespatchAdviceExportQuery = """ + SELECT + d.DespatchNumber AS despatch_id, + d.OrderReferenceId AS order_reference_id, + d.IssueDate AS issue_date, + d.IssueTime AS issue_time, + '2.1' AS ubl_version, + d.CustomizationId AS customization_id, + d.Note AS note, + d.SupplierPartyJson AS supplier_party_json, + d.DeliveryPartyJson AS delivery_party_json, + d.ShipmentJson AS shipment_json, + d.DespatchLinesJson AS despatch_lines_json + FROM DespatchAdvices d + WHERE d.Exported = 0 + ORDER BY d.DespatchNumber + """; + + // Trimmed from ERP_Example_-_Inventory_UBL_2.1_ScribanTemplate.md. + public const string DefaultInventoryExportQuery = """ + SELECT + r.ReportNumber AS report_id, + r.IssueDate AS issue_date, + r.PeriodStartDate AS period_start_date, + r.PeriodEndDate AS period_end_date, + '2.1' AS ubl_version, + r.CustomizationId AS customization_id, + r.RetailerPartyJson AS retailer_party_json, + r.LocationJson AS location_json, + r.InventoryLinesJson AS inventory_lines_json + FROM InventoryReports r + ORDER BY r.ReportNumber + """; + + // Mock rows for the ScribanTemplateVerifier when the staging/query table can't be probed + // for a real row (this recipe has no staging table at all) - shaped to match every field + // the four XML templates reference so a preview render always succeeds. + public static Dictionary MockInvoiceRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["invoice_id"] = "INV-2025-09-00123", + ["supplier_name"] = "ACME Consulting GmbH", + ["customer_name"] = "Global Innovations B.V.", + ["invoice_type_code"] = "380", + ["ubl_version"] = "2.1", + ["customization_id"] = "urn:cen.eu:en16931:2017#compliant#urn:fdc:peppol.eu:2017:poacc:billing:3.0", + ["issue_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["due_date"] = DateTime.UtcNow.AddDays(30).ToString("yyyy-MM-dd"), + ["currency"] = "EUR", + ["total_net"] = "1000.00", + ["total_tax"] = "190.00", + ["total_gross"] = "1190.00", + ["payment_means_code"] = "30", + ["payee_iban"] = "DE98765432101234567890", + ["supplier_party_json"] = """{"endpoint_id":"DE123456789","vat_id":"DE123456789","street":"Hauptstrasse 42","city":"Berlin","postal":"10115","country_code":"DE","contact_email":"billing@acme-consulting.de"}""", + ["customer_party_json"] = """{"endpoint_id":"NL987654321","vat_id":"NL987654321","street":"Keizersgracht 100","city":"Amsterdam","postal":"1015 CN","country_code":"NL","contact_email":"ap@global-innovations.nl"}""", + ["tax_summary_json"] = """[{"tax_rate":"19.0","taxable_amount":"1000.00","tax_amount":"190.00","category_code":"S"}]""", + ["line_items_json"] = """[{"id":1,"description":"Reef Demo Service","quantity":"20","unit_code":"HUR","unit_price":"40.00","net_total":"800.00","tax_rate":"19.0"},{"id":2,"description":"Reef Demo License","quantity":"1","unit_code":"EA","unit_price":"200.00","net_total":"200.00","tax_rate":"19.0"}]""" + }; + + public static Dictionary MockOrderRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["order_id"] = "PO-2025-10-00456", + ["issue_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["currency"] = "EUR", + ["note"] = "Urgent delivery required. Please confirm receipt.", + ["ubl_version"] = "2.1", + ["customization_id"] = "urn:cen.eu:en16931:2017#compliant#urn:fdc:peppol.eu:2017:poacc:billing:3.0", + ["total_net"] = "1200.00", + ["total_tax"] = "228.00", + ["total_gross"] = "1428.00", + ["buyer_party_json"] = """{"endpoint_id":"NL987654321","name":"Global Innovations B.V.","street":"Keizersgracht 100","city":"Amsterdam","postal":"1015 CN","country_code":"NL","contact_email":"purchasing@global-innovations.nl"}""", + ["seller_party_json"] = """{"endpoint_id":"DE123456789","name":"ACME Consulting GmbH","street":"Hauptstrasse 42","city":"Berlin","postal":"10115","country_code":"DE"}""", + ["delivery_json"] = $$"""{"street":"Warehouse Dock 4, Port Road 1","city":"Rotterdam","postal":"3011 AA","country_code":"NL","delivery_date":"{{DateTime.UtcNow.AddDays(14):yyyy-MM-dd}}"}""", + ["line_items_json"] = """[{"id":1,"description":"Reef Demo Server Blade","quantity":"2","unit_code":"C62","unit_price":"500.00","net_total":"1000.00"},{"id":2,"description":"Reef Demo Rack Kit","quantity":"2","unit_code":"EA","unit_price":"100.00","net_total":"200.00"}]""" + }; + + public static Dictionary MockDespatchAdviceRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["despatch_id"] = "DES-2025-09-9988", + ["order_reference_id"] = "PO-2025-10-00456", + ["issue_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["issue_time"] = DateTime.UtcNow.ToString("HH:mm:ss"), + ["ubl_version"] = "2.1", + ["customization_id"] = "urn:cen.eu:en16931:2017#compliant#urn:fdc:peppol.eu:2017:poacc:billing:3.0", + ["note"] = "Partial shipment of server blades", + ["supplier_party_json"] = """{"endpoint_id":"DE123456789","name":"ACME Consulting GmbH","street":"Hauptstrasse 42","city":"Berlin","postal":"10115","country_code":"DE"}""", + ["delivery_party_json"] = """{"endpoint_id":"NL987654321","name":"Global Innovations B.V.","street":"Warehouse Dock 4, Port Road 1","city":"Rotterdam","postal":"3011 AA","country_code":"NL"}""", + ["shipment_json"] = """{"gross_weight_measure":"45.5","weight_unit":"KGM","handling_code":"Fragile","carrier_name":"FastLogistics EU","tracking_id":"TRK-9988776655"}""", + ["despatch_lines_json"] = """[{"id":1,"item_name":"Reef Demo Server Blade","delivered_qty":"2","unit_code":"C62","order_line_id":"1","sellers_item_id":"SRV-BLD-09"},{"id":2,"item_name":"Reef Demo Rack Kit","delivered_qty":"2","unit_code":"EA","order_line_id":"2","sellers_item_id":"RCK-MNT-01"}]""" + }; + + public static Dictionary MockInventoryRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["report_id"] = "INV-RPT-2025-W04", + ["issue_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["period_start_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["period_end_date"] = DateTime.UtcNow.AddDays(7).ToString("yyyy-MM-dd"), + ["ubl_version"] = "2.1", + ["customization_id"] = "urn:cen.eu:en16931:2017#compliant#urn:fdc:peppol.eu:2017:poacc:billing:3.0", + ["retailer_party_json"] = """{"endpoint_id":"NL987654321","name":"Global Innovations B.V."}""", + ["location_json"] = """{"id":"WH-AMSTERDAM-01","name":"Main Distribution Center","street":"Logistics Way 12","city":"Amsterdam","postal":"1011 AZ","country_code":"NL"}""", + ["inventory_lines_json"] = """[{"id":1,"item_name":"Reef Demo Wireless Mouse","item_id":"WM-100-BLK","quantity":"450","unit_code":"EA","location_zone":"Aisle 4, Shelf B"},{"id":2,"item_name":"Reef Demo Keyboard","item_id":"MK-500-RGB","quantity":"120","unit_code":"EA","location_zone":"Aisle 4, Shelf C"}]""" + }; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "UBL Standard Export", + Description = "Export Invoices, Orders, Despatch Advices, or Inventory Reports from your own database to generic, vendor-neutral UBL 2.1 XML files - works with any ERP that can read standard UBL, not locked to one vendor. No import required, queries your own tables directly.", + Category = "ERP", + Icon = "file-text", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection + // Deliberately no CanAutoProvision: this Connection is the user's real business + // database, never a Reef-managed staging file. + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null, + CanAutoProvision = true + }, + + new() + { + StepKey = "invoice-query-template", + Title = "Invoice XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = InvoiceFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "invoice-export-profile", + Title = "Export Invoices to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = InvoiceFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "invoice-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = InvoiceFlowGroup, + VerifierKind = null + }, + + new() + { + StepKey = "order-query-template", + Title = "Order XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = OrderFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "order-export-profile", + Title = "Export Orders to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = OrderFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "order-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = OrderFlowGroup, + VerifierKind = null + }, + + new() + { + StepKey = "despatch-query-template", + Title = "Despatch Advice XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = DespatchFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "despatch-export-profile", + Title = "Export Despatch Advices to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = DespatchFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "despatch-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = DespatchFlowGroup, + VerifierKind = null + }, + + new() + { + StepKey = "inventory-query-template", + Title = "Inventory XML Template", + EntityType = RecipeEntityType.QueryTemplate, + FlowGroup = InventoryFlowGroup, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "inventory-export-profile", + Title = "Export Inventory Reports to XML", + EntityType = RecipeEntityType.Profile, + FlowGroup = InventoryFlowGroup, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "inventory-jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + FlowGroup = InventoryFlowGroup, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Recipes/UblExportTemplates.cs b/Source/Reef/Core/Recipes/UblExportTemplates.cs new file mode 100644 index 0000000..46ab060 --- /dev/null +++ b/Source/Reef/Core/Recipes/UblExportTemplates.cs @@ -0,0 +1,392 @@ +// AUTO-CONSTRUCTED constant - DO NOT hand-edit the body below. +// Source: /Templates/ERP_Example_-_Invoice_UBL_2.1_ScribanTemplate.xml, +// /Templates/ERP_Example_-_Order_UBL_2.1_ScribanTemplate.xml, +// /Templates/ERP_Example_-_Despatch_Advice_UBL_2.1_ScribanTemplate.xml and +// /Templates/ERP_Example_-_Inventory_UBL_2.1_ScribanTemplate.xml (reference copies retained +// on disk; these are the seed copies copied into QueryTemplates rows at step-execution time). +namespace Reef.Core.Recipes; + +internal static class UblExportTemplates +{ + public const string InvoiceXmlTemplate = """"""""""""""" +{{~ for row in rows ~}} + + + + + {{~ row.ubl_version | html.escape ~}} + {{~ row.customization_id | html.escape ~}} + + + {{~ row.invoice_id | html.escape ~}} + {{~ row.issue_date | html.escape ~}} + {{~ row.invoice_type_code | html.escape ~}} + {{~ row.currency | html.escape ~}} + + + + {{~ row.payment_means_code | html.escape ~}} + {{~ row.payment_means_code | html.escape ~}} + {{~ row.due_date | html.escape ~}} + + {{~ row.payee_iban | html.escape ~}} + + + + + {{~ supplier = parse_json(row.supplier_party_json) ~}} + + + {{~ supplier.endpoint_id | html.escape ~}} + {{~ row.supplier_name | html.escape ~}} + + {{~ supplier.street | html.escape ~}} + {{~ supplier.city | html.escape ~}} + {{~ supplier.postal | html.escape ~}} + {{~ supplier.country_code | html.escape ~}} + + + {{~ supplier.vat_id | html.escape ~}} + VAT + + {{~ supplier.contact_email | html.escape ~}} + + + + + {{~ customer = parse_json(row.customer_party_json) ~}} + + + {{~ customer.endpoint_id | html.escape ~}} + {{~ row.customer_name | html.escape ~}} + + {{~ customer.street | html.escape ~}} + {{~ customer.city | html.escape ~}} + {{~ customer.postal | html.escape ~}} + {{~ customer.country_code | html.escape ~}} + + + {{~ customer.vat_id | html.escape ~}} + VAT + + {{~ customer.contact_email | html.escape ~}} + + + + + {{~ tax_summary = parse_json(row.tax_summary_json) ~}} + + {{~ row.total_tax | html.escape ~}} + + {{~ for tax in tax_summary ~}} + + {{~ tax.taxable_amount | html.escape ~}} + {{~ tax.tax_amount | html.escape ~}} + + {{~ tax.category_code | html.escape ~}} + {{~ tax.tax_rate | html.escape ~}} + VAT + + + {{~ end ~}} + + + + + {{~ row.total_net | html.escape ~}} + {{~ row.total_net | html.escape ~}} + {{~ row.total_gross | html.escape ~}} + {{~ row.total_gross | html.escape ~}} + + + + {{~ line_items = parse_json(row.line_items_json) ~}} + {{~ for item in line_items ~}} + + {{~ item.id | html.escape ~}} + {{~ item.quantity | html.escape ~}} + {{~ item.net_total | html.escape ~}} + + {{~ item.description | html.escape ~}} + {{~ item.description | html.escape ~}} + + + {{~ item.unit_price | html.escape ~}} + + + {{ item.net_total | html.escape }} + + {{ item.net_total | html.escape }} + {{ item.tax_rate | html.escape }} + + S + {{~ item.tax_rate | html.escape ~}} + VAT + + + + + {{~ end ~}} + + +{{~ end ~}} +"""""""""""""""; + + public const string OrderXmlTemplate = """"""""""""""" +{{~ for row in rows ~}} + + + + {{~ row.ubl_version | html.escape ~}} + {{~ row.customization_id | html.escape ~}} + {{~ row.order_id | html.escape ~}} + {{~ row.issue_date | html.escape ~}} + {{~ row.note | html.escape ~}} + {{~ row.currency | html.escape ~}} + + + {{~ buyer = parse_json(row.buyer_party_json) ~}} + + + {{~ buyer.endpoint_id | html.escape ~}} + {{~ buyer.name | html.escape ~}} + + {{~ buyer.street | html.escape ~}} + {{~ buyer.city | html.escape ~}} + {{~ buyer.postal | html.escape ~}} + {{~ buyer.country_code | html.escape ~}} + + {{~ buyer.contact_email | html.escape ~}} + + + + + {{~ seller = parse_json(row.seller_party_json) ~}} + + + {{~ seller.endpoint_id | html.escape ~}} + {{~ seller.name | html.escape ~}} + + {{~ seller.street | html.escape ~}} + {{~ seller.city | html.escape ~}} + {{~ seller.postal | html.escape ~}} + {{~ seller.country_code | html.escape ~}} + + + + + + {{~ delivery = parse_json(row.delivery_json) ~}} + + + + {{~ delivery.street | html.escape ~}} + {{~ delivery.city | html.escape ~}} + {{~ delivery.postal | html.escape ~}} + {{~ delivery.country_code | html.escape ~}} + + + + {{~ delivery.delivery_date | html.escape ~}} + + + + + + {{~ row.total_tax | html.escape ~}} + + + + + {{~ row.total_net | html.escape ~}} + {{~ row.total_net | html.escape ~}} + {{~ row.total_gross | html.escape ~}} + {{~ row.total_gross | html.escape ~}} + + + + {{~ line_items = parse_json(row.line_items_json) ~}} + {{~ for item in line_items ~}} + + + {{~ item.id | html.escape ~}} + {{~ item.quantity | html.escape ~}} + {{~ item.net_total | html.escape ~}} + + {{~ item.unit_price | html.escape ~}} + + + {{~ item.description | html.escape ~}} + + + + {{~ end ~}} + + +{{~ end ~}} +"""""""""""""""; + + public const string DespatchAdviceXmlTemplate = """"""""""""""" +{{~ for row in rows ~}} + + + + {{~ row.ubl_version | html.escape ~}} + {{~ row.customization_id | html.escape ~}} + {{~ row.despatch_id | html.escape ~}} + {{~ row.issue_date | html.escape ~}} + {{~ row.issue_time | html.escape ~}} + 351 + {{~ row.note | html.escape ~}} + + + + {{~ row.order_reference_id | html.escape ~}} + + + + {{~ supplier = parse_json(row.supplier_party_json) ~}} + + + {{~ supplier.endpoint_id | html.escape ~}} + {{~ supplier.name | html.escape ~}} + + {{~ supplier.street | html.escape ~}} + {{~ supplier.city | html.escape ~}} + {{~ supplier.postal | html.escape ~}} + {{~ supplier.country_code | html.escape ~}} + + + + + + {{~ delivery = parse_json(row.delivery_party_json) ~}} + + + {{~ delivery.endpoint_id | html.escape ~}} + {{~ delivery.name | html.escape ~}} + + {{~ delivery.street | html.escape ~}} + {{~ delivery.city | html.escape ~}} + {{~ delivery.postal | html.escape ~}} + {{~ delivery.country_code | html.escape ~}} + + + + + + {{~ ship = parse_json(row.shipment_json) ~}} + + SHP-{{~ row.despatch_id | html.escape ~}} + {{~ ship.gross_weight_measure | html.escape ~}} + + {{~ ship.tracking_id | html.escape ~}} + + + {{~ ship.carrier_name | html.escape ~}} + + + + + + + {{~ lines = parse_json(row.despatch_lines_json) ~}} + {{~ for line in lines ~}} + + {{~ line.id | html.escape ~}} + {{~ line.delivered_qty | html.escape ~}} + + {{~ line.order_line_id | html.escape ~}} + + + {{~ line.item_name | html.escape ~}} + + {{~ line.sellers_item_id | html.escape ~}} + + + + {{~ end ~}} + + +{{~ end ~}} +"""""""""""""""; + + public const string InventoryXmlTemplate = """"""""""""""" +{{~ for row in rows ~}} + + + + {{~ row.ubl_version | html.escape ~}} + {{~ row.customization_id | html.escape ~}} + {{~ row.report_id | html.escape ~}} + {{~ row.issue_date | html.escape ~}} + EUR + + + + {{~ row.period_start_date | html.escape ~}} + {{~ row.period_end_date | html.escape ~}} + + + + {{~ retailer = parse_json(row.retailer_party_json) ~}} + + + {{~ retailer.endpoint_id | html.escape ~}} + {{~ retailer.name | html.escape ~}} + + + + + {{~ loc = parse_json(row.location_json) ~}} + + {{~ loc.id | html.escape ~}} + {{~ loc.name | html.escape ~}} + + {{~ loc.street | html.escape ~}} + {{~ loc.city | html.escape ~}} + {{~ loc.postal | html.escape ~}} + {{~ loc.country_code | html.escape ~}} + + + + + {{~ lines = parse_json(row.inventory_lines_json) ~}} + {{~ for item in lines ~}} + + {{~ item.id | html.escape ~}} + {{~ item.quantity | html.escape ~}} + 0.00 + + {{~ item.item_name | html.escape ~}} + {{~ item.item_name | html.escape ~}} + + {{~ item.item_id | html.escape ~}} + + + + {{~ item.location_zone | html.escape ~}} + + + {{~ end ~}} + + +{{~ end ~}} +"""""""""""""""; +} diff --git a/Source/Reef/Core/Recipes/Verifiers.cs b/Source/Reef/Core/Recipes/Verifiers.cs new file mode 100644 index 0000000..8e8121f --- /dev/null +++ b/Source/Reef/Core/Recipes/Verifiers.cs @@ -0,0 +1,271 @@ +using Reef.Core.Models; +using Reef.Core.Services; +using Reef.Core.Sources; +using Reef.Core.Targets; +using Reef.Core.TemplateEngines; + +namespace Reef.Core.Recipes; + +// Wraps ConnectionService.TestConnectionAsync (same mechanism as ConnectionsEndpoints' /test). +public class ConnectionVerifier : IRecipeVerifier +{ + private readonly ConnectionService _connectionService; + private readonly EncryptionService _encryptionService; + + public ConnectionVerifier(ConnectionService connectionService, EncryptionService encryptionService) + { + _connectionService = connectionService; + _encryptionService = encryptionService; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.EntityId is not { } connectionId) + return new RecipeVerifyResult { Success = false, Message = "No Connection has been saved for this step yet." }; + + var connection = await _connectionService.GetByIdAsync(connectionId, ct); + if (connection is null) + return new RecipeVerifyResult { Success = false, Message = $"Connection {connectionId} not found." }; + + // GetByIdAsync returns the connection string encrypted; decrypt before testing. + var connectionString = _encryptionService.IsEncrypted(connection.ConnectionString) + ? _encryptionService.Decrypt(connection.ConnectionString) + : connection.ConnectionString; + + var (success, message, responseTimeMs) = await _connectionService.TestConnectionAsync(connection.Type, connectionString, ct); + + return new RecipeVerifyResult + { + Success = success, + Message = message ?? (success ? "Connection successful." : "Connection test failed."), + Detail = $"{responseTimeMs}ms" + }; + } +} + +// Wraps ImportSourceFactory + IImportSource.TestAsync (same path as ImportProfilesEndpoints.TestSource). +public class HttpSourceVerifier : IRecipeVerifier +{ + private readonly ImportProfileService _importProfileService; + + public HttpSourceVerifier(ImportProfileService importProfileService) + { + _importProfileService = importProfileService; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.EntityId is not { } importProfileId) + return new RecipeVerifyResult { Success = false, Message = "No ImportProfile has been saved for this step yet." }; + + var profile = await _importProfileService.GetByIdAsync(importProfileId); + if (profile is null) + return new RecipeVerifyResult { Success = false, Message = $"ImportProfile {importProfileId} not found." }; + + try + { + var source = ImportSourceFactory.Create(profile.SourceType); + var (success, message) = await source.TestAsync(profile, ct); + return new RecipeVerifyResult { Success = success, Message = message ?? (success ? "Source reachable." : "Source test failed.") }; + } + catch (Exception ex) + { + return new RecipeVerifyResult { Success = false, Message = $"Source test failed: {ex.Message}" }; + } + } +} + +// Wraps DatabaseImportTarget.TestAsync's table-exists check. +public class StagingTableVerifier : IRecipeVerifier +{ + private readonly ConnectionService _connectionService; + private readonly DatabaseImportTarget _databaseImportTarget; + + public StagingTableVerifier(ConnectionService connectionService, DatabaseImportTarget databaseImportTarget) + { + _connectionService = connectionService; + _databaseImportTarget = databaseImportTarget; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.ConnectionId is not { } connectionId) + return new RecipeVerifyResult { Success = false, Message = "No Connection selected for the staging table." }; + + if (string.IsNullOrWhiteSpace(context.TableName)) + return new RecipeVerifyResult { Success = false, Message = "No staging table name configured." }; + + var connection = await _connectionService.GetByIdAsync(connectionId, ct); + if (connection is null) + return new RecipeVerifyResult { Success = false, Message = $"Connection {connectionId} not found." }; + + var (success, message) = await _databaseImportTarget.TestAsync(connection, context.TableName, ct); + return new RecipeVerifyResult { Success = success, Message = message ?? (success ? "Table exists." : "Table check failed.") }; + } +} + +// Wraps DestinationService.TestDestinationConfigurationAsync - a real send, not just a probe. +public class EmailDestinationVerifier : IRecipeVerifier +{ + private readonly DestinationService _destinationService; + + public EmailDestinationVerifier(DestinationService destinationService) + { + _destinationService = destinationService; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.EntityId is not { } destinationId) + return new RecipeVerifyResult { Success = false, Message = "No Email destination has been saved for this step yet." }; + + // Needs the fully-decrypted (never masked) config to actually send the test email. + var destination = await _destinationService.GetByIdForExecutionAsync(destinationId, ct); + if (destination is null) + return new RecipeVerifyResult { Success = false, Message = $"Destination {destinationId} not found." }; + + var result = await _destinationService.TestDestinationConfigurationAsync( + destination.Type, + destination.ConfigurationJson); + + return new RecipeVerifyResult + { + Success = result.Success, + Message = result.Message ?? (result.Success ? "Test email sent." : "Test email failed."), + Detail = result.ResponseTimeMs is { } ms ? $"{ms}ms" : null + }; + } +} + +// Renders against one real staging row, or mock data if the table is empty - same +// Scriban preview path QueryTemplateEndpoints' /preview handler uses. +public class ScribanTemplateVerifier : IRecipeVerifier +{ + private readonly QueryTemplateService _queryTemplateService; + private readonly ConnectionService _connectionService; + private readonly QueryExecutor _queryExecutor; + private readonly ScribanTemplateEngine _templateEngine; + + public ScribanTemplateVerifier( + QueryTemplateService queryTemplateService, + ConnectionService connectionService, + QueryExecutor queryExecutor, + ScribanTemplateEngine templateEngine) + { + _queryTemplateService = queryTemplateService; + _connectionService = connectionService; + _queryExecutor = queryExecutor; + _templateEngine = templateEngine; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.EntityId is not { } templateId) + return new RecipeVerifyResult { Success = false, Message = "No QueryTemplate has been saved for this step yet." }; + + var template = await _queryTemplateService.GetByIdAsync(templateId); + if (template is null) + return new RecipeVerifyResult { Success = false, Message = $"QueryTemplate {templateId} not found." }; + + var rows = await TryFetchOneStagingRowAsync(context, ct) ?? new List> { context.MockTemplateRow ?? MockOrderRow() }; + + try + { + var preview = await _templateEngine.TransformAsync(rows, template.Template); + var truncated = preview.Length > 500 ? preview[..500] + "…" : preview; + return new RecipeVerifyResult { Success = true, Message = "Template rendered successfully.", Detail = truncated }; + } + catch (Exception ex) + { + return new RecipeVerifyResult { Success = false, Message = $"Template render failed: {ex.Message}" }; + } + } + + private async Task>?> TryFetchOneStagingRowAsync(RecipeVerifyContext context, CancellationToken ct) + { + if (context.ConnectionId is not { } connectionId || string.IsNullOrWhiteSpace(context.TableName)) + return null; + + var connection = await _connectionService.GetByIdAsync(connectionId, ct); + if (connection is null) return null; + + // SQL Server uses TOP; MySQL, PostgreSQL, and SQLite all support LIMIT. + var sql = connection.Type == "SqlServer" + ? $"SELECT TOP 1 * FROM {context.TableName}" + : $"SELECT * FROM {context.TableName} LIMIT 1"; + + var (success, rows, _, _) = await _queryExecutor.ExecuteQueryAsync(connection, sql, null, commandTimeout: 15); + return success && rows.Count > 0 ? rows : null; + } + + private static Dictionary MockOrderRow() => new(StringComparer.OrdinalIgnoreCase) + { + ["order_number"] = "1001", + ["order_date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"), + ["order_status"] = "PROCESSING", + ["customer_name"] = "Jane Doe", + ["company"] = "Reef Demo Store", + ["currency_symbol"] = "$", + ["total"] = "129.99", + ["subtotal"] = "119.99", + ["shipping"] = "10.00", + ["items_json"] = "[{\"product_name\":\"Sample Product\",\"sku\":\"SKU-1\",\"quantity\":1,\"unit_price\":\"119.99\",\"line_total\":\"119.99\"}]" + }; +} + +// Wraps the same QueryExecutor path ProfilesEndpoints' TestQuery/TestQueryPreview use. +public class ExportQueryVerifier : IRecipeVerifier +{ + private readonly ConnectionService _connectionService; + private readonly QueryExecutor _queryExecutor; + + public ExportQueryVerifier(ConnectionService connectionService, QueryExecutor queryExecutor) + { + _connectionService = connectionService; + _queryExecutor = queryExecutor; + } + + public async Task VerifyAsync(RecipeVerifyContext context, CancellationToken ct = default) + { + if (context.ConnectionId is not { } connectionId) + return new RecipeVerifyResult { Success = false, Message = "No Connection selected for the export query." }; + + if (context.Params.GetValueOrDefault("query") is not string query || string.IsNullOrWhiteSpace(query)) + return new RecipeVerifyResult { Success = false, Message = "No export query configured for this step." }; + + var connection = await _connectionService.GetByIdAsync(connectionId, ct); + if (connection is null) + return new RecipeVerifyResult { Success = false, Message = $"Connection {connectionId} not found." }; + + var testQuery = query.Trim(); + var hasLimit = testQuery.Contains("LIMIT", StringComparison.OrdinalIgnoreCase) + || testQuery.Contains("TOP", StringComparison.OrdinalIgnoreCase) + || testQuery.Contains("FETCH", StringComparison.OrdinalIgnoreCase); + + if (!hasLimit) + { + if (connection.Type == "SqlServer") + { + var selectIndex = testQuery.IndexOf("SELECT", StringComparison.OrdinalIgnoreCase); + if (selectIndex >= 0) testQuery = testQuery.Insert(selectIndex + "SELECT".Length, " TOP 25"); + } + else + { + // MySQL, PostgreSQL, and SQLite all support LIMIT. + testQuery += " LIMIT 25"; + } + } + + var (success, rows, error, executionTime) = await _queryExecutor.ExecuteQueryAsync(connection, testQuery, null, commandTimeout: 30); + + if (!success) + return new RecipeVerifyResult { Success = false, Message = error ?? "Query failed." }; + + return new RecipeVerifyResult + { + Success = true, + Message = $"Query returned {rows.Count} row(s) in {executionTime}ms.", + Detail = rows.Count == 0 ? "No rows matched yet - this is expected if no orders have synced." : null + }; + } +} diff --git a/Source/Reef/Core/Recipes/WooCommerceEmailTemplates.cs b/Source/Reef/Core/Recipes/WooCommerceEmailTemplates.cs new file mode 100644 index 0000000..247e470 --- /dev/null +++ b/Source/Reef/Core/Recipes/WooCommerceEmailTemplates.cs @@ -0,0 +1,524 @@ +// AUTO-CONSTRUCTED constant - DO NOT hand-edit the body below. +// Source: /Templates/WooCommerce_-_Email_Order_Confirmation.ScribanTemplate.html (reference copy retained on disk; this is the seed copied into a QueryTemplates row at step-execution time) +namespace Reef.Core.Recipes; + +internal static class WooCommerceEmailTemplates +{ + public const string OrderConfirmationHtml = """"""""""""""" +{{~ for row in rows ~}} + + + + + + Your order {{~ row.order_number | html.escape ~}} is confirmed + + + + + + +{{~ end ~}} + +"""""""""""""""; + + // Source: /Templates/WooCommerce_-_Email_Tracking_Update.ScribanTemplate.html (reference copy retained on disk; this is the seed copied into a QueryTemplates row at step-execution time) + public const string TrackingUpdateHtml = """"""""""""""" +{{~ for row in rows ~}} + + + + + + A shipment for your order {{~ row.order_number | html.escape ~}} is on its way + + + + + + +{{~ end ~}} +"""""""""""""""; +} diff --git a/Source/Reef/Core/Recipes/WooCommerceRecipe.cs b/Source/Reef/Core/Recipes/WooCommerceRecipe.cs new file mode 100644 index 0000000..05def74 --- /dev/null +++ b/Source/Reef/Core/Recipes/WooCommerceRecipe.cs @@ -0,0 +1,271 @@ +namespace Reef.Core.Recipes; + +// Order Confirmation only - Tracking Link split out into its own WooCommerceTrackingRecipe +// (separate RecipeRun, separate Connection/Group/Destination) since the two were never +// genuinely one setup, just two flows that happened to share infrastructure. The shipments +// staging-table DDL statics stay here (rather than duplicated into WooCommerceTrackingRecipe) +// since they're identical to before the split and WooCommerceTrackingRecipe references them. +public static class WooCommerceRecipe +{ + public const string Key = "woocommerce-order-confirmation"; + public const string StagingTableName = "StoreOrders"; + public const string ShipmentsStagingTableName = "StoreShipments"; + + // Per-database-type identity/text syntax for Reef's supported query sources. + // Sqlite here is the user's chosen staging file (never Reef's own app database). + public static string GetStagingTableDdl(string connectionType) => connectionType switch + { + "SqlServer" => """ + CREATE TABLE StoreOrders ( + Id INT IDENTITY(1,1) PRIMARY KEY, + WooOrderId BIGINT NOT NULL, + OrderNumber NVARCHAR(50) NULL, + OrderDate DATETIME NULL, + OrderStatus NVARCHAR(50) NULL, + CustomerName NVARCHAR(255) NULL, + CustomerEmail NVARCHAR(255) NULL, + Company NVARCHAR(255) NULL, + CurrencySymbol NVARCHAR(10) NULL, + Total DECIMAL(18,2) NULL, + Subtotal DECIMAL(18,2) NULL, + Discount DECIMAL(18,2) NULL, + Shipping DECIMAL(18,2) NULL, + Tax DECIMAL(18,2) NULL, + ShippingAddressJson NVARCHAR(MAX) NULL, + BillingAddressJson NVARCHAR(MAX) NULL, + ItemsJson NVARCHAR(MAX) NULL, + EmailSent BIT NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT GETUTCDATE(), + CONSTRAINT UQ_StoreOrders_WooOrderId UNIQUE (WooOrderId) + ) + """, + "MySQL" or "MariaDB" => """ + CREATE TABLE StoreOrders ( + Id INT AUTO_INCREMENT PRIMARY KEY, + WooOrderId BIGINT NOT NULL, + OrderNumber VARCHAR(50) NULL, + OrderDate DATETIME NULL, + OrderStatus VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + CurrencySymbol VARCHAR(10) NULL, + Total DECIMAL(18,2) NULL, + Subtotal DECIMAL(18,2) NULL, + Discount DECIMAL(18,2) NULL, + Shipping DECIMAL(18,2) NULL, + Tax DECIMAL(18,2) NULL, + ShippingAddressJson TEXT NULL, + BillingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent TINYINT(1) NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY UQ_StoreOrders_WooOrderId (WooOrderId) + ) + """, + "Sqlite" => """ + CREATE TABLE StoreOrders ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + WooOrderId INTEGER NOT NULL, + OrderNumber TEXT NULL, + OrderDate TEXT NULL, + OrderStatus TEXT NULL, + CustomerName TEXT NULL, + CustomerEmail TEXT NULL, + Company TEXT NULL, + CurrencySymbol TEXT NULL, + Total TEXT NULL, + Subtotal TEXT NULL, + Discount TEXT NULL, + Shipping TEXT NULL, + Tax TEXT NULL, + ShippingAddressJson TEXT NULL, + BillingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent INTEGER NOT NULL DEFAULT 0, + CreatedAt TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (WooOrderId) + ) + """, + _ => """ + CREATE TABLE StoreOrders ( + Id SERIAL PRIMARY KEY, + WooOrderId BIGINT NOT NULL UNIQUE, + OrderNumber VARCHAR(50) NULL, + OrderDate TIMESTAMP NULL, + OrderStatus VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + CurrencySymbol VARCHAR(10) NULL, + Total DECIMAL(18,2) NULL, + Subtotal DECIMAL(18,2) NULL, + Discount DECIMAL(18,2) NULL, + Shipping DECIMAL(18,2) NULL, + Tax DECIMAL(18,2) NULL, + ShippingAddressJson TEXT NULL, + BillingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP NOT NULL DEFAULT NOW() + ) + """ + }; + + public const string DefaultExportQuery = "SELECT * FROM StoreOrders WHERE EmailSent = 0"; + + public static string GetShipmentsStagingTableDdl(string connectionType) => connectionType switch + { + "SqlServer" => """ + CREATE TABLE StoreShipments ( + Id INT IDENTITY(1,1) PRIMARY KEY, + WooOrderId BIGINT NOT NULL, + OrderNumber NVARCHAR(50) NULL, + CustomerName NVARCHAR(255) NULL, + CustomerEmail NVARCHAR(255) NULL, + Company NVARCHAR(255) NULL, + TrackingNumber NVARCHAR(100) NULL, + Carrier NVARCHAR(100) NULL, + TrackingUrl NVARCHAR(500) NULL, + EstimatedDeliveryDate NVARCHAR(50) NULL, + ShippingAddressJson NVARCHAR(MAX) NULL, + ItemsJson NVARCHAR(MAX) NULL, + EmailSent BIT NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT GETUTCDATE(), + CONSTRAINT UQ_StoreShipments_WooOrderId UNIQUE (WooOrderId) + ) + """, + "MySQL" or "MariaDB" => """ + CREATE TABLE StoreShipments ( + Id INT AUTO_INCREMENT PRIMARY KEY, + WooOrderId BIGINT NOT NULL, + OrderNumber VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + TrackingNumber VARCHAR(100) NULL, + Carrier VARCHAR(100) NULL, + TrackingUrl VARCHAR(500) NULL, + EstimatedDeliveryDate VARCHAR(50) NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent TINYINT(1) NOT NULL DEFAULT 0, + CreatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY UQ_StoreShipments_WooOrderId (WooOrderId) + ) + """, + "Sqlite" => """ + CREATE TABLE StoreShipments ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + WooOrderId INTEGER NOT NULL, + OrderNumber TEXT NULL, + CustomerName TEXT NULL, + CustomerEmail TEXT NULL, + Company TEXT NULL, + TrackingNumber TEXT NULL, + Carrier TEXT NULL, + TrackingUrl TEXT NULL, + EstimatedDeliveryDate TEXT NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent INTEGER NOT NULL DEFAULT 0, + CreatedAt TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (WooOrderId) + ) + """, + _ => """ + CREATE TABLE StoreShipments ( + Id SERIAL PRIMARY KEY, + WooOrderId BIGINT NOT NULL UNIQUE, + OrderNumber VARCHAR(50) NULL, + CustomerName VARCHAR(255) NULL, + CustomerEmail VARCHAR(255) NULL, + Company VARCHAR(255) NULL, + TrackingNumber VARCHAR(100) NULL, + Carrier VARCHAR(100) NULL, + TrackingUrl VARCHAR(500) NULL, + EstimatedDeliveryDate VARCHAR(50) NULL, + ShippingAddressJson TEXT NULL, + ItemsJson TEXT NULL, + EmailSent BOOLEAN NOT NULL DEFAULT FALSE, + CreatedAt TIMESTAMP NOT NULL DEFAULT NOW() + ) + """ + }; + + public const string DefaultShipmentsExportQuery = "SELECT * FROM StoreShipments WHERE EmailSent = 0"; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "WooCommerce Order Confirmation", + Description = "Pull new orders from your WooCommerce store and automatically email customers a confirmation.", + Category = "E-commerce", + Icon = "shopping-cart", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection, + CanAutoProvision = true + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null, + CanAutoProvision = true + }, + new() + { + StepKey = "destination", + Title = "Email Destination", + EntityType = RecipeEntityType.Destination, + IsShared = true, + VerifierKind = RecipeVerifierKind.EmailDestination + }, + new() + { + StepKey = "staging-table", + Title = "Staging Table", + EntityType = RecipeEntityType.StagingTable, + VerifierKind = RecipeVerifierKind.StagingTable + }, + new() + { + StepKey = "import-profile", + Title = "Pull Orders from WooCommerce", + EntityType = RecipeEntityType.ImportProfile, + VerifierKind = RecipeVerifierKind.HttpSource + }, + new() + { + StepKey = "query-template", + Title = "Email Template", + EntityType = RecipeEntityType.QueryTemplate, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "export-profile", + Title = "Send Confirmation Emails", + EntityType = RecipeEntityType.Profile, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "jobs", + Title = "Scheduling (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Recipes/WooCommerceTrackingRecipe.cs b/Source/Reef/Core/Recipes/WooCommerceTrackingRecipe.cs new file mode 100644 index 0000000..c521c59 --- /dev/null +++ b/Source/Reef/Core/Recipes/WooCommerceTrackingRecipe.cs @@ -0,0 +1,84 @@ +namespace Reef.Core.Recipes; + +// Single-flow recipe (split out of WooCommerceRecipe's former Flow B) - pulls shipment +// tracking data from WooCommerce's REST API and emails customers an update. Step keys are +// plain (no "shipments-" prefix) since there's only one flow here, mirroring MagentoRecipe's +// shape exactly - RecipeService dispatches on recipe.Key instead. +public static class WooCommerceTrackingRecipe +{ + public const string Key = "woocommerce-tracking-link"; + + public static RecipeDefinition Definition { get; } = new() + { + Key = Key, + Name = "WooCommerce Tracking Link", + Description = "Pull shipment tracking data from your WooCommerce store's REST API and automatically email customers a tracking-link update.", + Category = "E-commerce", + Icon = "truck", + Steps = new List + { + new() + { + StepKey = "connection", + Title = "Database Connection", + EntityType = RecipeEntityType.Connection, + IsShared = true, + VerifierKind = RecipeVerifierKind.Connection, + CanAutoProvision = true + }, + new() + { + StepKey = "group", + Title = "Organize", + EntityType = RecipeEntityType.Group, + IsShared = true, + VerifierKind = null, + CanAutoProvision = true + }, + new() + { + StepKey = "destination", + Title = "Email Destination", + EntityType = RecipeEntityType.Destination, + IsShared = true, + VerifierKind = RecipeVerifierKind.EmailDestination + }, + new() + { + StepKey = "staging-table", + Title = "Staging Table", + EntityType = RecipeEntityType.StagingTable, + VerifierKind = RecipeVerifierKind.StagingTable + }, + new() + { + StepKey = "import-profile", + Title = "Pull Tracking Updates from WooCommerce", + EntityType = RecipeEntityType.ImportProfile, + VerifierKind = RecipeVerifierKind.HttpSource + }, + new() + { + StepKey = "query-template", + Title = "Email Template", + EntityType = RecipeEntityType.QueryTemplate, + VerifierKind = RecipeVerifierKind.ScribanTemplate + }, + new() + { + StepKey = "export-profile", + Title = "Send Tracking Emails", + EntityType = RecipeEntityType.Profile, + VerifierKind = RecipeVerifierKind.ExportQuery + }, + new() + { + StepKey = "jobs", + Title = "Scheduling & Webhook (optional)", + EntityType = RecipeEntityType.Job, + IsOptional = true, + VerifierKind = null + } + } + }; +} diff --git a/Source/Reef/Core/Scripting/IScriptRunner.cs b/Source/Reef/Core/Scripting/IScriptRunner.cs new file mode 100644 index 0000000..7276f50 --- /dev/null +++ b/Source/Reef/Core/Scripting/IScriptRunner.cs @@ -0,0 +1,73 @@ +namespace Reef.Core.Scripting; + +/// +/// Request to execute a script as a child process. +/// The script receives on stdin and is expected to +/// write its result to stdout. Stderr is captured for diagnostics only. +/// +public class ScriptExecutionRequest +{ + /// + /// Interpreter to use: pwsh, python, node, bash, sh, cmd + /// + public required string Interpreter { get; init; } + + /// + /// Either a path to an existing script file, or the inline script source + /// (see ). + /// + public required string ScriptPathOrInline { get; init; } + + /// + /// When true, is the script source and is + /// written to a temp file before execution. When false it is a path. + /// + public bool IsInline { get; init; } + + /// + /// JSON payload written to the process stdin. + /// + public required string StdinJson { get; init; } + + /// + /// Max time to let the process run before it (and its process tree) is killed. + /// + public int TimeoutSeconds { get; init; } = 30; + + /// + /// Names of environment variables to copy from the current process into the + /// child process. The child otherwise gets a minimal environment (PATH only). + /// + public List? EnvAllowlist { get; init; } + + /// + /// Working directory for the script process. Defaults to a temp directory. + /// + public string? WorkingDirectory { get; init; } + + /// + /// Whether to log a warning when the interpreter binary itself can't be + /// started (missing on this host). Real Profile/Import Script steps + /// always want this (a missing interpreter is the cause of a real + /// failure). InterpreterService's routine availability probe - run on + /// every Profile editor page load - sets this false to avoid logging the + /// same "pwsh not installed" warning on every load; it logs its own + /// summary once instead. + /// + public bool LogIfUnavailable { get; init; } = true; +} + +public class ScriptExecutionResult +{ + public required bool Success { get; init; } + public int ExitCode { get; init; } + public string Stdout { get; init; } = ""; + public string Stderr { get; init; } = ""; + public string? ErrorMessage { get; init; } + public long ElapsedMs { get; init; } +} + +public interface IScriptRunner +{ + Task RunAsync(ScriptExecutionRequest request, CancellationToken ct = default); +} diff --git a/Source/Reef/Core/Scripting/ProcessScriptRunner.cs b/Source/Reef/Core/Scripting/ProcessScriptRunner.cs new file mode 100644 index 0000000..eae5acf --- /dev/null +++ b/Source/Reef/Core/Scripting/ProcessScriptRunner.cs @@ -0,0 +1,226 @@ +using System.ComponentModel; +using System.Diagnostics; +using Serilog; + +namespace Reef.Core.Scripting; + +/// +/// Runs scripts as child processes (no shell), feeding context as JSON on stdin +/// and capturing stdout/stderr/exit code. Cross-platform interpreter resolution, +/// process-tree kill on timeout, and a minimal inherited environment so secrets +/// in the host process env don't leak into the script unless explicitly allowed. +/// +public class ProcessScriptRunner : IScriptRunner +{ + private static readonly Serilog.ILogger Log = Serilog.Log.ForContext(); + + public async Task RunAsync(ScriptExecutionRequest request, CancellationToken ct = default) + { + var stopwatch = Stopwatch.StartNew(); + string? tempScriptPath = null; + + try + { + var scriptPath = request.IsInline + ? tempScriptPath = WriteInlineScript(request.Interpreter, request.ScriptPathOrInline) + : request.ScriptPathOrInline; + + if (!request.IsInline && !File.Exists(scriptPath)) + { + return new ScriptExecutionResult + { + Success = false, + ExitCode = -1, + ErrorMessage = $"Script file not found: {scriptPath}", + ElapsedMs = stopwatch.ElapsedMilliseconds + }; + } + + var (fileName, arguments) = ResolveInterpreter(request.Interpreter, scriptPath); + + var psi = new ProcessStartInfo + { + FileName = fileName, + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = request.WorkingDirectory ?? Path.GetTempPath() + }; + foreach (var arg in arguments) + psi.ArgumentList.Add(arg); + + ApplyMinimalEnvironment(psi, request.EnvAllowlist); + + using var process = new Process { StartInfo = psi }; + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(Math.Max(1, request.TimeoutSeconds))); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token); + + process.Start(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(linkedCts.Token); + var stderrTask = process.StandardError.ReadToEndAsync(linkedCts.Token); + + try + { + await process.StandardInput.WriteAsync(request.StdinJson.AsMemory(), linkedCts.Token); + } + catch (IOException) + { + // Script may not read stdin at all (broken pipe) - not fatal. + } + finally + { + try { process.StandardInput.Close(); } catch (IOException) { /* already closed by the child */ } + } + + try + { + await process.WaitForExitAsync(linkedCts.Token); + } + catch (OperationCanceledException) + { + KillProcessTree(process); + + var timedOut = timeoutCts.IsCancellationRequested; + stopwatch.Stop(); + return new ScriptExecutionResult + { + Success = false, + ExitCode = -1, + Stdout = await SafeAwait(stdoutTask), + Stderr = await SafeAwait(stderrTask), + ErrorMessage = timedOut + ? $"Script timed out after {request.TimeoutSeconds}s and was killed" + : "Script execution was cancelled", + ElapsedMs = stopwatch.ElapsedMilliseconds + }; + } + + var stdout = await stdoutTask; + var stderr = await stderrTask; + stopwatch.Stop(); + + return new ScriptExecutionResult + { + Success = process.ExitCode == 0, + ExitCode = process.ExitCode, + Stdout = stdout, + Stderr = stderr, + ErrorMessage = process.ExitCode == 0 ? null : $"Script exited with code {process.ExitCode}", + ElapsedMs = stopwatch.ElapsedMilliseconds + }; + } + catch (Win32Exception ex) + { + // Interpreter binary not found/not runnable (e.g. pwsh/cmd missing + // on this host). One line, no stack trace - and only logged when + // the caller wants it (see LogIfUnavailable doc comment). + stopwatch.Stop(); + if (request.LogIfUnavailable) + Log.Warning("Interpreter '{Interpreter}' unavailable: {Message}", request.Interpreter, ex.Message); + return new ScriptExecutionResult + { + Success = false, + ExitCode = -1, + ErrorMessage = $"Interpreter '{request.Interpreter}' unavailable: {ex.Message}", + ElapsedMs = stopwatch.ElapsedMilliseconds + }; + } + catch (Exception ex) + { + stopwatch.Stop(); + Log.Error(ex, "Script execution failed: {Message}", ex.Message); + return new ScriptExecutionResult + { + Success = false, + ExitCode = -1, + ErrorMessage = $"Script execution exception: {ex.Message}", + ElapsedMs = stopwatch.ElapsedMilliseconds + }; + } + finally + { + if (tempScriptPath != null) + { + try { File.Delete(tempScriptPath); } catch { /* best effort cleanup */ } + } + } + } + + private static async Task SafeAwait(Task task) + { + try { return await task; } catch { return ""; } + } + + private static void KillProcessTree(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + // Process may have exited between the check and the kill - ignore. + } + } + + private static string WriteInlineScript(string interpreter, string source) + { + var ext = interpreter.ToLowerInvariant() switch + { + "pwsh" or "powershell" => ".ps1", + "python" or "python3" => ".py", + "node" or "nodejs" => ".js", + "bash" => ".sh", + "sh" => ".sh", + "cmd" => ".bat", + _ => ".txt" + }; + + var path = Path.Combine(Path.GetTempPath(), $"reef-script-{Guid.NewGuid():N}{ext}"); + File.WriteAllText(path, source); + return path; + } + + private static (string FileName, List Arguments) ResolveInterpreter(string interpreter, string scriptPath) + { + return interpreter.ToLowerInvariant() switch + { + "pwsh" or "powershell" => ("pwsh", new List { "-NoProfile", "-NonInteractive", "-File", scriptPath }), + "python" or "python3" => ("python3", new List { scriptPath }), + "node" or "nodejs" => ("node", new List { scriptPath }), + "bash" => ("bash", new List { scriptPath }), + "sh" => ("sh", new List { scriptPath }), + "cmd" => ("cmd", new List { "/c", scriptPath }), + _ => throw new ArgumentException($"Unsupported script interpreter: {interpreter}") + }; + } + + private static void ApplyMinimalEnvironment(ProcessStartInfo psi, List? envAllowlist) + { + psi.EnvironmentVariables.Clear(); + + var path = Environment.GetEnvironmentVariable("PATH"); + if (!string.IsNullOrEmpty(path)) + psi.EnvironmentVariables["PATH"] = path; + + if (OperatingSystem.IsWindows()) + { + var systemRoot = Environment.GetEnvironmentVariable("SystemRoot"); + if (!string.IsNullOrEmpty(systemRoot)) + psi.EnvironmentVariables["SystemRoot"] = systemRoot; + } + + if (envAllowlist == null) return; + + foreach (var name in envAllowlist) + { + var value = Environment.GetEnvironmentVariable(name); + if (value != null) + psi.EnvironmentVariables[name] = value; + } + } +} diff --git a/Source/Reef/Core/Services/ConnectionService.cs b/Source/Reef/Core/Services/ConnectionService.cs index dfb74a6..11d3753 100644 --- a/Source/Reef/Core/Services/ConnectionService.cs +++ b/Source/Reef/Core/Services/ConnectionService.cs @@ -68,6 +68,8 @@ public async Task> GetAllAsync(CancellationToken ct = default) /// public async Task CreateAsync(Connection conn, int? createdByUserId, CancellationToken ct = default) { + conn.Tags ??= ""; // column is NOT NULL DEFAULT '' + await using var connection = new SqliteConnection(_connectionString); await connection.OpenAsync(ct); @@ -252,7 +254,11 @@ private DbConnection CreateDatabaseConnection(string type, string connectionStri { ApplicationName = "Reef" }.ConnectionString), - + + // Sqlite connections point at a dedicated staging file chosen by the caller + // (e.g. "Data Source=/path/to/file.db"). This is NEVER Reef's own app database. + "Sqlite" => new Microsoft.Data.Sqlite.SqliteConnection(connectionString), + _ => throw new NotSupportedException($"Database type {type} is not supported") }; } diff --git a/Source/Reef/Core/Services/ExecutionService.cs b/Source/Reef/Core/Services/ExecutionService.cs index 15b1703..6eeea71 100644 --- a/Source/Reef/Core/Services/ExecutionService.cs +++ b/Source/Reef/Core/Services/ExecutionService.cs @@ -30,6 +30,7 @@ public class ExecutionService private readonly EmailExportService _emailExportService; private readonly EmailApprovalService _emailApprovalService; private readonly NotificationService _notificationService; + private readonly Reef.Core.Scripting.IScriptRunner _scriptRunner; private readonly Microsoft.Extensions.Configuration.IConfiguration _configuration; @@ -46,6 +47,7 @@ public ExecutionService( EmailExportService emailExportService, EmailApprovalService emailApprovalService, NotificationService notificationService, + Reef.Core.Scripting.IScriptRunner scriptRunner, Microsoft.Extensions.Configuration.IConfiguration configuration) { _connectionString = config.ConnectionString; @@ -62,6 +64,7 @@ public ExecutionService( _emailExportService = emailExportService; _emailApprovalService = emailApprovalService; _notificationService = notificationService; + _scriptRunner = scriptRunner; } /// @@ -162,7 +165,8 @@ await UpdateExecutionRecordAsync( Status = "Running", ErrorMessage = null, DeltaSyncReefIdColumn = profile.DeltaSyncReefIdColumn, - SplitKeyColumn = profile.SplitKeyColumn + SplitKeyColumn = profile.SplitKeyColumn, + WebhookParameters = parameters }; var (preProcessSuccess, preProcessError) = await ExecutePreProcessingAsync( @@ -299,7 +303,8 @@ await UpdateExecutionRecordAsync(executionId, "Failed", originalRowCount, null, Status = "Success", ErrorMessage = null, DeltaSyncReefIdColumn = profile.DeltaSyncReefIdColumn, - SplitKeyColumn = profile.SplitKeyColumn + SplitKeyColumn = profile.SplitKeyColumn, + WebhookParameters = parameters }; var zeroRowConnection = await _connectionService.GetByIdAsync(profile.ConnectionId); @@ -703,7 +708,8 @@ await UpdateExecutionWithSplitSummaryAsync( Status = splitSuccess ? "Success" : "Failed", ErrorMessage = splitError, DeltaSyncReefIdColumn = profile.DeltaSyncReefIdColumn, - SplitKeyColumn = profile.SplitKeyColumn + SplitKeyColumn = profile.SplitKeyColumn, + WebhookParameters = parameters }; var splitConnection = await _connectionService.GetByIdAsync(profile.ConnectionId); if (splitConnection != null) @@ -1266,7 +1272,8 @@ await UpdateExecutionRecordAsync(executionId, "Failed", rows.Count, null, stopwa Status = "Success", ErrorMessage = null, DeltaSyncReefIdColumn = profile.DeltaSyncReefIdColumn, - SplitKeyColumn = profile.SplitKeyColumn + SplitKeyColumn = profile.SplitKeyColumn, + WebhookParameters = parameters }; // Execute new post-processing logic (supports both Query and StoredProcedure) @@ -1830,6 +1837,30 @@ UPDATE ProfileExecutions return (false, errorMsg); } + if (config.Type.Equals("Script", StringComparison.OrdinalIgnoreCase)) + { + var (scriptSuccess, scriptError, stdout, stderr, exitCode) = await RunProcessingScriptAsync(config, context); + stopwatch.Stop(); + + if (scriptSuccess) + { + Log.Debug("Pre-processing script completed successfully in {ElapsedMs}ms", stopwatch.ElapsedMilliseconds); + await UpdatePreProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Success", null, stopwatch.ElapsedMilliseconds, stdout, stderr, exitCode); + return (true, null); + } + + Log.Warning("Pre-processing script failed: {Error}", scriptError); + await UpdatePreProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Failed", scriptError, stopwatch.ElapsedMilliseconds, stdout, stderr, exitCode); + + if (!config.ContinueOnError) + { + return (false, scriptError); + } + + Log.Information("Continuing execution despite pre-processing script error (ContinueOnError=true)"); + return (true, null); + } + // Build and execute command var command = BuildDatabaseCommand(connection.Type, config, context); Log.Debug("Executing pre-processing command: {Command}", command); @@ -1841,7 +1872,7 @@ UPDATE ProfileExecutions if (success) { - Log.Debug("Pre-processing completed successfully in {ElapsedMs}ms. Rows affected: {RowsAffected}", + Log.Debug("Pre-processing completed successfully in {ElapsedMs}ms. Rows affected: {RowsAffected}", stopwatch.ElapsedMilliseconds, rowsAffected); await UpdatePreProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Success", null, stopwatch.ElapsedMilliseconds); return (true, null); @@ -1850,12 +1881,12 @@ UPDATE ProfileExecutions { Log.Warning("Pre-processing failed: {Error}", error); await UpdatePreProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Failed", error, stopwatch.ElapsedMilliseconds); - + if (!config.ContinueOnError) { return (false, error); } - + Log.Information("Continuing execution despite pre-processing error (ContinueOnError=true)"); return (true, null); // Continue execution } @@ -1928,6 +1959,30 @@ await UpdatePostProcessingStatusAsync(executionId, DateTime.UtcNow, DateTime.Utc return (false, errorMsg); } + if (config.Type.Equals("Script", StringComparison.OrdinalIgnoreCase)) + { + var (scriptSuccess, scriptError, stdout, stderr, exitCode) = await RunProcessingScriptAsync(config, context); + stopwatch.Stop(); + + if (scriptSuccess) + { + Log.Debug("Post-processing script completed successfully in {ElapsedMs}ms", stopwatch.ElapsedMilliseconds); + await UpdatePostProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Success", null, stopwatch.ElapsedMilliseconds, stdout, stderr, exitCode); + return (true, null); + } + + Log.Warning("Post-processing script failed: {Error}", scriptError); + await UpdatePostProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Failed", scriptError, stopwatch.ElapsedMilliseconds, stdout, stderr, exitCode); + + if (!config.ContinueOnError) + { + return (false, scriptError); + } + + Log.Information("Ignoring post-processing script error (ContinueOnError=true)"); + return (true, null); + } + // Build and execute command var command = BuildDatabaseCommand(connection.Type, config, context); Log.Debug("Executing post-processing command: {Command}", command); @@ -1939,7 +1994,7 @@ await UpdatePostProcessingStatusAsync(executionId, DateTime.UtcNow, DateTime.Utc if (success) { - Log.Debug("Post-processing completed successfully in {ElapsedMs}ms. Rows affected: {RowsAffected}", + Log.Debug("Post-processing completed successfully in {ElapsedMs}ms. Rows affected: {RowsAffected}", stopwatch.ElapsedMilliseconds, rowsAffected); await UpdatePostProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Success", null, stopwatch.ElapsedMilliseconds); return (true, null); @@ -1948,12 +2003,12 @@ await UpdatePostProcessingStatusAsync(executionId, DateTime.UtcNow, DateTime.Utc { Log.Warning("Post-processing failed: {Error}", error); await UpdatePostProcessingStatusAsync(executionId, startedAt, DateTime.UtcNow, "Failed", error, stopwatch.ElapsedMilliseconds); - + if (!config.ContinueOnError) { return (false, error); } - + Log.Information("Ignoring post-processing error (ContinueOnError=true)"); return (true, null); // Don't fail overall execution } @@ -1978,6 +2033,9 @@ await UpdatePostProcessingStatusAsync(executionId, DateTime.UtcNow, DateTime.Utc /// SQL command string private string BuildDatabaseCommand(string connectionType, ProcessingConfig config, ProcessingContext context) { + if (string.IsNullOrWhiteSpace(config.Command)) + throw new ArgumentException($"ProcessingConfig.Command is required for Type={config.Type}"); + // Substitute context variables in the command var command = SubstituteContextVariables(config.Command, context); @@ -2083,6 +2141,53 @@ private Dictionary BuildParametersDictionary( return parameters; } + /// + /// Run a pre/post-processing step whose ProcessingConfig.Type is "Script". + /// Builds the stdin JSON context (execution context + webhook/job trigger + /// parameters) and executes it via IScriptRunner. + /// + private async Task<(bool Success, string? ErrorMessage, string Stdout, string Stderr, int ExitCode)> RunProcessingScriptAsync( + ProcessingConfig config, + ProcessingContext context) + { + if (string.IsNullOrWhiteSpace(config.Interpreter) || string.IsNullOrWhiteSpace(config.ScriptPathOrInline)) + { + return (false, "Script processing requires Interpreter and ScriptPathOrInline to be set", "", "", -1); + } + + var stdinPayload = System.Text.Json.JsonSerializer.Serialize(new + { + context.ExecutionId, + context.ProfileId, + context.RowCount, + context.OutputPath, + context.FileSizeBytes, + context.ExecutionTimeMs, + context.OutputFormat, + context.TriggeredBy, + context.StartedAt, + context.CompletedAt, + context.Status, + context.ErrorMessage, + context.DeltaSyncReefIdColumn, + context.SplitKeyColumn, + context.SplitKey, + parameters = context.WebhookParameters ?? new Dictionary() + }); + + var result = await _scriptRunner.RunAsync(new Reef.Core.Scripting.ScriptExecutionRequest + { + Interpreter = config.Interpreter, + ScriptPathOrInline = config.ScriptPathOrInline, + IsInline = config.ScriptIsInline, + StdinJson = stdinPayload, + TimeoutSeconds = config.Timeout, + EnvAllowlist = config.EnvAllowlist + }); + + return (result.Success, result.ErrorMessage, result.Stdout, result.Stderr, result.ExitCode); + } + /// /// Update pre-processing status in ProfileExecutions table /// @@ -2092,18 +2197,24 @@ private async Task UpdatePreProcessingStatusAsync( DateTime? completedAt, string status, string? error, - long? timeMs) + long? timeMs, + string? stdout = null, + string? stderr = null, + int? exitCode = null) { await using var connection = new SqliteConnection(_connectionString); await connection.OpenAsync(); const string sql = @" - UPDATE ProfileExecutions + UPDATE ProfileExecutions SET PreProcessStartedAt = @StartedAt, PreProcessCompletedAt = @CompletedAt, PreProcessStatus = @Status, PreProcessError = @Error, - PreProcessTimeMs = @TimeMs + PreProcessTimeMs = @TimeMs, + PreProcessStdout = @Stdout, + PreProcessStderr = @Stderr, + PreProcessExitCode = @ExitCode WHERE Id = @Id"; await connection.ExecuteAsync(sql, new @@ -2113,7 +2224,10 @@ UPDATE ProfileExecutions CompletedAt = completedAt, Status = status, Error = error, - TimeMs = timeMs + TimeMs = timeMs, + Stdout = stdout, + Stderr = stderr, + ExitCode = exitCode }); } @@ -2126,18 +2240,24 @@ private async Task UpdatePostProcessingStatusAsync( DateTime? completedAt, string status, string? error, - long? timeMs) + long? timeMs, + string? stdout = null, + string? stderr = null, + int? exitCode = null) { await using var connection = new SqliteConnection(_connectionString); await connection.OpenAsync(); const string sql = @" - UPDATE ProfileExecutions + UPDATE ProfileExecutions SET PostProcessStartedAt = @StartedAt, PostProcessCompletedAt = @CompletedAt, PostProcessStatus = @Status, PostProcessError = @Error, - PostProcessTimeMs = @TimeMs + PostProcessTimeMs = @TimeMs, + PostProcessStdout = @Stdout, + PostProcessStderr = @Stderr, + PostProcessExitCode = @ExitCode WHERE Id = @Id"; await connection.ExecuteAsync(sql, new @@ -2147,7 +2267,10 @@ UPDATE ProfileExecutions CompletedAt = completedAt, Status = status, Error = error, - TimeMs = timeMs + TimeMs = timeMs, + Stdout = stdout, + Stderr = stderr, + ExitCode = exitCode }); } diff --git a/Source/Reef/Core/Services/ImportExecutionService.cs b/Source/Reef/Core/Services/ImportExecutionService.cs index 1909579..2a729dc 100644 --- a/Source/Reef/Core/Services/ImportExecutionService.cs +++ b/Source/Reef/Core/Services/ImportExecutionService.cs @@ -6,6 +6,7 @@ using Microsoft.Data.Sqlite; using Reef.Core.Models; using Reef.Core.Parsers; +using Reef.Core.Scripting; using Reef.Core.Sources; using Reef.Core.Targets; using Serilog.Context; @@ -35,7 +36,8 @@ public class ImportExecutionService( LocalFileImportTarget localFileImportTarget, NotificationService notificationService, DestinationSourceAdapter sourceAdapter, - ImportFileOutputService fileOutputService) + ImportFileOutputService fileOutputService, + IScriptRunner scriptRunner) { private static readonly Serilog.ILogger Log = Serilog.Log.ForContext(); @@ -159,7 +161,7 @@ await TimedPhaseAsync("PreProcess", exec, phaseTimings, async () => { await RunSqlProcessAsync( connection!.Type, decryptedConnStr!, - profile.PreProcessConfig, exec.Id, ct); + profile.PreProcessConfig, exec, "Pre", ct); }, ct); } @@ -482,7 +484,7 @@ await TimedPhaseAsync("PostProcess", exec, phaseTimings, async () => { await RunSqlProcessAsync( connection!.Type, decryptedConnStr!, - profile.PostProcessConfig, exec.Id, ct); + profile.PostProcessConfig, exec, "Post", ct); } catch (Exception ex) { @@ -830,12 +832,28 @@ await conn.ExecuteAsync( // Pre/Post SQL Processing─ - private static async Task RunSqlProcessAsync( + private async Task RunSqlProcessAsync( string dbType, string connectionString, - string? processConfig, int execId, CancellationToken ct) + string? processConfig, ImportProfileExecution exec, string phase, CancellationToken ct) { if (string.IsNullOrWhiteSpace(processConfig)) return; + // Try the rich ProcessingConfig shape first (shared with Profile pre/post-processing). + // A Script type runs as a child process instead of against the target database. + try + { + var config = JsonSerializer.Deserialize(processConfig); + if (config != null && config.Type.Equals("Script", StringComparison.OrdinalIgnoreCase)) + { + await RunImportProcessingScriptAsync(config, exec, phase, ct); + return; + } + } + catch (JsonException) + { + // Not the ProcessingConfig shape - fall through to legacy raw-SQL handling below. + } + string? sql = null; try { @@ -852,13 +870,50 @@ private static async Task RunSqlProcessAsync( if (string.IsNullOrWhiteSpace(sql)) return; // Replace placeholder tokens - sql = sql.Replace("{ExecutionId}", execId.ToString()); + sql = sql.Replace("{ExecutionId}", exec.Id.ToString()); await using var dbConn = CreateDbConnection(dbType, connectionString); await dbConn.OpenAsync(ct); await dbConn.ExecuteAsync(new CommandDefinition(sql, cancellationToken: ct)); } + private async Task RunImportProcessingScriptAsync(ProcessingConfig config, ImportProfileExecution exec, string phase, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(config.Interpreter) || string.IsNullOrWhiteSpace(config.ScriptPathOrInline)) + throw new InvalidOperationException("Script processing requires Interpreter and ScriptPathOrInline to be set"); + + var stdinPayload = JsonSerializer.Serialize(new { ExecutionId = exec.Id }); + + var result = await scriptRunner.RunAsync(new ScriptExecutionRequest + { + Interpreter = config.Interpreter, + ScriptPathOrInline = config.ScriptPathOrInline, + IsInline = config.ScriptIsInline, + StdinJson = stdinPayload, + TimeoutSeconds = config.Timeout, + EnvAllowlist = config.EnvAllowlist + }, ct); + + Log.Debug("Import processing script for execution {ExecutionId} exited {ExitCode}. Stdout: {Stdout} Stderr: {Stderr}", + exec.Id, result.ExitCode, result.Stdout, result.Stderr); + + if (phase == "Pre") + { + exec.PreProcessStdout = result.Stdout; + exec.PreProcessStderr = result.Stderr; + exec.PreProcessExitCode = result.ExitCode; + } + else + { + exec.PostProcessStdout = result.Stdout; + exec.PostProcessStderr = result.Stderr; + exec.PostProcessExitCode = result.ExitCode; + } + + if (!result.Success && !config.ContinueOnError) + throw new InvalidOperationException($"Import processing script failed: {result.ErrorMessage ?? result.Stderr}"); + } + // Utility private static async Task TimedPhaseAsync( diff --git a/Source/Reef/Core/Services/ImportProfileService.cs b/Source/Reef/Core/Services/ImportProfileService.cs index c420d98..257ba42 100644 --- a/Source/Reef/Core/Services/ImportProfileService.cs +++ b/Source/Reef/Core/Services/ImportProfileService.cs @@ -300,7 +300,13 @@ UPDATE ImportProfileExecutions SET DeltaSyncUnchangedRows = @DeltaSyncUnchangedRows, DeltaSyncDeletedRows = @DeltaSyncDeletedRows, PhaseTimingsJson = @PhaseTimingsJson, - OutputTargetsJson = @OutputTargetsJson + OutputTargetsJson = @OutputTargetsJson, + PreProcessStdout = @PreProcessStdout, + PreProcessStderr = @PreProcessStderr, + PreProcessExitCode = @PreProcessExitCode, + PostProcessStdout = @PostProcessStdout, + PostProcessStderr = @PostProcessStderr, + PostProcessExitCode = @PostProcessExitCode WHERE Id = @Id"; var rows = await conn.ExecuteAsync(sql, exec); diff --git a/Source/Reef/Core/Services/InterpreterService.cs b/Source/Reef/Core/Services/InterpreterService.cs new file mode 100644 index 0000000..e677ce7 --- /dev/null +++ b/Source/Reef/Core/Services/InterpreterService.cs @@ -0,0 +1,93 @@ +using System.Collections.Concurrent; +using Reef.Core.Scripting; +using Serilog; + +namespace Reef.Core.Services; + +/// +/// Probes whether the script interpreters used by Profile pre/post-process +/// Script steps are actually runnable on this host. Runs each interpreter +/// through the real IScriptRunner path with a trivial "print ok" script - +/// the same execution path production Script steps use - rather than just +/// checking PATH for the binary name, so a "yes" here means scripts will +/// actually run, not just that something with that name exists. +/// +public class InterpreterService(IScriptRunner scriptRunner) +{ + private static readonly Serilog.ILogger Log = Serilog.Log.ForContext(); + + // CheckAllAsync runs on every Profile editor page load (it's how the UI + // greys out unavailable interpreters) - logging "pwsh unavailable" on + // every single load would spam the log for something that's static + // per-host. Logged once per interpreter per process lifetime instead; + // an explicit manual re-check (the Admin "Test" button) always logs. + private static readonly ConcurrentDictionary _loggedUnavailable = new(); + + public static readonly IReadOnlyList KnownInterpreters = new List + { + new("pwsh", "PowerShell", "Write-Output 'ok'"), + new("python", "Python", "print('ok')"), + new("node", "Node.js", "console.log('ok')"), + new("bash", "Bash", "echo ok"), + new("sh", "sh", "echo ok"), + new("cmd", "cmd (Windows)", "echo ok"), + }; + + public async Task> CheckAllAsync(CancellationToken ct = default) + { + var results = new List(); + foreach (var def in KnownInterpreters) + { + var status = await CheckAsync(def.Name, logIfUnavailable: false, ct); + if (!status.Available && _loggedUnavailable.TryAdd(def.Name, true)) + Log.Warning("Interpreter '{Interpreter}' unavailable: {Error}", def.Name, status.Error); + + results.Add(status); + } + return results; + } + + /// Explicit single re-check (Admin's "Test" button) - always logs. + public Task CheckOneAsync(string name, CancellationToken ct = default) => + CheckAsync(name, logIfUnavailable: true, ct); + + private async Task CheckAsync(string name, bool logIfUnavailable, CancellationToken ct) + { + var def = KnownInterpreters.FirstOrDefault(d => + d.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); + + if (def == null) + { + return new InterpreterStatus(name, name, false, null, $"Unknown interpreter: {name}"); + } + + var result = await scriptRunner.RunAsync(new ScriptExecutionRequest + { + Interpreter = def.Name, + ScriptPathOrInline = def.ProbeScript, + IsInline = true, + StdinJson = "{}", + TimeoutSeconds = 10, + LogIfUnavailable = logIfUnavailable + }, ct); + + var available = result.Success && result.Stdout.Contains("ok", StringComparison.Ordinal); + if (available) _loggedUnavailable.TryRemove(def.Name, out _); + + return new InterpreterStatus( + def.Name, + def.DisplayName, + available, + available ? result.Stdout.Trim() : null, + available ? null : (result.ErrorMessage ?? result.Stderr.Trim())); + } +} + +public record InterpreterDefinition(string Name, string DisplayName, string ProbeScript); + +public record InterpreterStatus( + string Name, + string DisplayName, + bool Available, + string? Output, + string? Error); diff --git a/Source/Reef/Core/Services/QueryExecutor.cs b/Source/Reef/Core/Services/QueryExecutor.cs index 0f7d900..ec2fc8a 100644 --- a/Source/Reef/Core/Services/QueryExecutor.cs +++ b/Source/Reef/Core/Services/QueryExecutor.cs @@ -1,4 +1,5 @@ using Dapper; +using Microsoft.Data.Sqlite; using Microsoft.Data.SqlClient; using MySqlConnector; using Npgsql; @@ -11,7 +12,7 @@ namespace Reef.Core.Services; /// /// Service for executing SQL queries against different database types -/// Supports SQL Server, MySQL, and PostgreSQL with native parameterization +/// Supports SQL Server, MySQL, PostgreSQL, and SQLite with native parameterization /// public class QueryExecutor { @@ -187,6 +188,10 @@ public QueryExecutor(EncryptionService encryptionService, ValidationService vali rows = await ExecutePostgreSqlQueryAsync(connectionString, query, parameters, commandTimeout); break; + case "sqlite": + rows = await ExecuteSqliteQueryAsync(connectionString, query, parameters, commandTimeout); + break; + default: return (false, new List>(), $"Unsupported database type: {connection.Type}", stopwatch.ElapsedMilliseconds); @@ -309,7 +314,11 @@ private bool IsTransientPostgreSqlError(Npgsql.NpgsqlException ex) case "postgres": rowsAffected = await ExecutePostgreSqlCommandAsync(connectionString, command, parameters, commandTimeout); break; - + + case "sqlite": + rowsAffected = await ExecuteSqliteCommandAsync(connectionString, command, parameters, commandTimeout); + break; + default: return (false, 0, $"Unsupported database type: {connection.Type}", stopwatch.ElapsedMilliseconds); } @@ -379,6 +388,19 @@ private async Task ExecutePostgreSqlCommandAsync( return await connection.ExecuteAsync(command, paramObject, commandTimeout: commandTimeout); } + /// + /// Execute command against SQLite using ExecuteAsync with native parameters + /// + private async Task ExecuteSqliteCommandAsync( + string connectionString, string command, Dictionary? parameters, int commandTimeout) + { + await using var connection = new SqliteConnection(connectionString); + await connection.OpenAsync(); + + object? paramObject = parameters != null && parameters.Count > 0 ? new DynamicParameters(parameters) : null; + return await connection.ExecuteAsync(command, paramObject, commandTimeout: commandTimeout); + } + /// /// Execute query against SQL Server /// @@ -427,6 +449,18 @@ private async Task>> ExecutePostgreSqlQueryAsync return await ExecuteQueryWithDapper(connection, query, parameters, commandTimeout); } + /// + /// Execute query against SQLite + /// + private async Task>> ExecuteSqliteQueryAsync( + string connectionString, string query, Dictionary? parameters, int commandTimeout) + { + await using var connection = new SqliteConnection(connectionString); + await connection.OpenAsync(); + + return await ExecuteQueryWithDapper(connection, query, parameters, commandTimeout); + } + /// /// Execute query using Dapper with native parameterization and convert results to dictionary list /// @@ -528,7 +562,17 @@ private async Task>> ExecuteQueryWithDapper( await conn.ExecuteAsync("SELECT 1"); } break; - + + case "sqlite": + { + // Opening a SqliteConnection creates the file automatically if it doesn't + // exist yet - this is SQLite's normal behavior, not an error case. + await using var conn = new SqliteConnection(connectionString); + await conn.OpenAsync(); + await conn.ExecuteAsync("SELECT 1"); + } + break; + default: return (false, $"Unsupported database type: {connection.Type}", stopwatch.ElapsedMilliseconds); } diff --git a/Source/Reef/Core/Services/RecipeService.cs b/Source/Reef/Core/Services/RecipeService.cs new file mode 100644 index 0000000..b24f38d --- /dev/null +++ b/Source/Reef/Core/Services/RecipeService.cs @@ -0,0 +1,1152 @@ +using System.Text.Json; +using Dapper; +using Microsoft.Data.Sqlite; +using Reef.Core.Models; +using Reef.Core.Recipes; +using Serilog; + +namespace Reef.Core.Services; + +// Pure orchestration for the Store wizard - delegates every create/update to the existing +// entity services. The one new piece of logic is the staging-table DDL step: Reef's import +// pipeline never auto-creates target tables, so the wizard issues CREATE TABLE itself. +public class RecipeService +{ + // Stable, recognizable name for the auto-provisioned Sqlite staging Connection - "find" + // half of find-or-create. A user who renames it breaks the auto-reuse (they've opted out + // by touching it), which is an acceptable edge case for a name-based marker. + public const string StagingConnectionName = "Store Staging Database"; + private const string StagingDbFileName = "store-staging.db"; + + private readonly string _connectionString; + private readonly ConnectionService _connectionService; + private readonly GroupService _groupService; + private readonly DestinationService _destinationService; + private readonly ImportProfileService _importProfileService; + private readonly QueryTemplateService _queryTemplateService; + private readonly ProfileService _profileService; + private readonly JobService _jobService; + private readonly WebhookService _webhookService; + private readonly EncryptionService _encryptionService; + private readonly IReadOnlyDictionary _verifiers; + private static readonly Serilog.ILogger Log = Serilog.Log.ForContext(); + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true }; + + public RecipeService( + DatabaseConfig config, + ConnectionService connectionService, + GroupService groupService, + DestinationService destinationService, + ImportProfileService importProfileService, + QueryTemplateService queryTemplateService, + ProfileService profileService, + JobService jobService, + WebhookService webhookService, + EncryptionService encryptionService, + ConnectionVerifier connectionVerifier, + HttpSourceVerifier httpSourceVerifier, + StagingTableVerifier stagingTableVerifier, + EmailDestinationVerifier emailDestinationVerifier, + ScribanTemplateVerifier scribanTemplateVerifier, + ExportQueryVerifier exportQueryVerifier) + { + _connectionString = config.ConnectionString; + _connectionService = connectionService; + _groupService = groupService; + _destinationService = destinationService; + _importProfileService = importProfileService; + _queryTemplateService = queryTemplateService; + _profileService = profileService; + _jobService = jobService; + _webhookService = webhookService; + _encryptionService = encryptionService; + + _verifiers = new Dictionary + { + [RecipeVerifierKind.Connection] = connectionVerifier, + [RecipeVerifierKind.HttpSource] = httpSourceVerifier, + [RecipeVerifierKind.StagingTable] = stagingTableVerifier, + [RecipeVerifierKind.EmailDestination] = emailDestinationVerifier, + [RecipeVerifierKind.ScribanTemplate] = scribanTemplateVerifier, + [RecipeVerifierKind.ExportQuery] = exportQueryVerifier + }; + } + + public async Task> GetAvailableRecipesAsync(int? userId, CancellationToken ct = default) + { + var runs = await GetRunsForUserAsync(userId, ct); + + return RecipeRegistry.All.Select(recipe => + { + var existingRun = runs + .Where(r => r.RecipeKey.Equals(recipe.Key, StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(r => r.UpdatedAt) + .FirstOrDefault(); + + // Independent of existingRun so Reconfigure stays offered even after a newer + // InProgress run exists for the same recipe. + var lastCompletedRun = runs + .Where(r => r.RecipeKey.Equals(recipe.Key, StringComparison.OrdinalIgnoreCase) && r.Status == "Completed") + .OrderByDescending(r => r.CompletedAt) + .FirstOrDefault(); + + return new RecipeListItem + { + Key = recipe.Key, + Name = recipe.Name, + Description = recipe.Description, + Category = recipe.Category, + Icon = recipe.Icon, + StepCount = recipe.Steps.Count, + ExistingRunId = existingRun?.Status == "InProgress" ? existingRun.Id : null, + LastCompletedRunId = lastCompletedRun?.Id, + IsInstalled = lastCompletedRun is not null + }; + }).ToList(); + } + + private async Task> GetRunsForUserAsync(int? userId, CancellationToken ct) + { + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + const string sql = "SELECT * FROM RecipeRuns WHERE (@UserId IS NULL AND CreatedBy IS NULL) OR CreatedBy = @UserId ORDER BY UpdatedAt DESC"; + var rows = await conn.QueryAsync(new CommandDefinition(sql, new { UserId = userId }, cancellationToken: ct)); + return rows.ToList(); + } + + // cloneFromRunId (the "Reconfigure" action) pre-seeds the new run's StepStateJson from a + // prior Completed run's entity ids/params, but never the Verified flag - pointing an + // existing entity at new credentials means it's unverified until re-checked. + public async Task StartRecipeAsync(string recipeKey, int? userId, int? cloneFromRunId = null, CancellationToken ct = default) + { + var recipe = RecipeRegistry.GetByKey(recipeKey) + ?? throw new InvalidOperationException($"Unknown recipe '{recipeKey}'."); + + var seedStepState = await BuildCloneSeedStateAsync(recipe, cloneFromRunId, ct); + await ApplyAutoProvisioningAsync(recipe, seedStepState, userId, ct); + + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + var now = DateTime.UtcNow; + const string sql = @" + INSERT INTO RecipeRuns (RecipeKey, Status, CurrentStepKey, StepStateJson, CreatedAt, UpdatedAt, CreatedBy) + VALUES (@RecipeKey, 'InProgress', @CurrentStepKey, @StepStateJson, @CreatedAt, @UpdatedAt, @CreatedBy); + SELECT last_insert_rowid();"; + + var id = await conn.ExecuteScalarAsync(new CommandDefinition(sql, new + { + RecipeKey = recipe.Key, + // Skip past any already-auto-provisioned leading steps - simple mode hides them + // from the rail entirely, so landing the wizard on one of them on first load would + // show a step the user can't navigate to. Falls back to the very first step if + // every step happens to be auto-provisioned (defensive; not true for any recipe today). + CurrentStepKey = (recipe.Steps.FirstOrDefault(s => !s.CanAutoProvision || !seedStepState.ContainsKey(s.StepKey)) ?? recipe.Steps.First()).StepKey, + StepStateJson = JsonSerializer.Serialize(seedStepState, JsonOptions), + CreatedAt = now, + UpdatedAt = now, + CreatedBy = userId + }, cancellationToken: ct)); + + Log.Information( + "Recipe run started: {RecipeKey} (RunId: {RunId}) by user {UserId}{ClonedFrom}", + recipe.Key, id, userId, cloneFromRunId is { } cf ? $" (cloned from run {cf})" : ""); + return await GetRunOrThrowAsync(id, ct); + } + + private async Task> BuildCloneSeedStateAsync(RecipeDefinition recipe, int? cloneFromRunId, CancellationToken ct) + { + if (cloneFromRunId is not { } sourceRunId) + return new(); + + var sourceRun = await GetRunOrThrowAsync(sourceRunId, ct); + if (!sourceRun.RecipeKey.Equals(recipe.Key, StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException($"Run {sourceRunId} belongs to a different recipe and cannot be used to reconfigure '{recipe.Key}'."); + + var sourceState = DeserializeStepState(sourceRun.StepStateJson); + var seeded = new Dictionary(); + + foreach (var step in recipe.Steps) + { + if (sourceState.GetValueOrDefault(step.StepKey) is not { } sourceStepState) + continue; + + seeded[step.StepKey] = new RecipeStepState + { + EntityId = sourceStepState.EntityId, + Params = new Dictionary(sourceStepState.Params), + Verified = false, + Skipped = sourceStepState.Skipped + }; + } + + return seeded; + } + + // "Simple mode" - for every CanAutoProvision step the run doesn't already have a value + // for (fresh start, or not carried over by Reconfigure-cloning), silently find-or-create + // a sensible default entity and seed it into the run's initial StepStateJson as Verified. + // These are real Connections/Groups the user can find on the normal pages afterward - + // "simple mode" only means the wizard doesn't ask up front. The Advanced toggle in + // store.js reveals these steps again so a user can override with their own entity, which + // just flows through the existing ExecuteStepAsync save path like any manual step. + private async Task ApplyAutoProvisioningAsync(RecipeDefinition recipe, Dictionary seedStepState, int? userId, CancellationToken ct) + { + foreach (var step in recipe.Steps) + { + if (!step.CanAutoProvision || seedStepState.ContainsKey(step.StepKey)) + continue; + + int? entityId = step.EntityType switch + { + RecipeEntityType.Connection => await FindOrCreateStagingConnectionAsync(userId, ct), + RecipeEntityType.Group => await FindOrCreateRecipeGroupAsync(recipe, ct), + _ => null + }; + + if (entityId is null) + continue; + + seedStepState[step.StepKey] = new RecipeStepState + { + EntityId = entityId, + Params = AutoProvisionedParams(recipe, step, entityId.Value), + // We control exactly what we just created/found, so it doesn't need the live + // ConnectionVerifier check a user-supplied entity would - mark it Verified so + // the rail's "done" status and canAdvance() work without the user clicking. + Verified = true, + Skipped = false + }; + + Log.Information("Recipe '{RecipeKey}' auto-provisioned {EntityType} {EntityId} for step '{StepKey}'", + recipe.Key, step.EntityType, entityId, step.StepKey); + } + } + + private static Dictionary AutoProvisionedParams(RecipeDefinition recipe, RecipeStepDefinition step, int entityId) => step.EntityType switch + { + RecipeEntityType.Connection => new Dictionary + { + ["name"] = StagingConnectionName, + ["type"] = "Sqlite", + ["existingConnectionId"] = entityId + }, + RecipeEntityType.Group => new Dictionary + { + ["name"] = $"{recipe.Name} (Store)" + }, + _ => new Dictionary() + }; + + // Find-or-create by the stable StagingConnectionName marker - idempotent and reusable + // across every recipe run/recipe that opts into auto-provisioning, never a duplicate per run. + private async Task FindOrCreateStagingConnectionAsync(int? userId, CancellationToken ct) + { + var existing = await _connectionService.GetByNameAsync(StagingConnectionName, ct); + if (existing is not null) + return existing.Id; + + var path = ResolveStagingDbPath(); + var newConn = new Connection + { + Name = StagingConnectionName, + Type = "Sqlite", + ConnectionString = $"Data Source={path}", + Tags = "", + Hash = "" + }; + + var id = await _connectionService.CreateAsync(newConn, userId, ct); + Log.Information("Auto-provisioned Sqlite staging Connection '{Name}' (ID: {Id}) at {Path}", StagingConnectionName, id, path); + return id; + } + + // Sibling file next to Reef's own app database (never that file itself) - e.g. app db at + // /data/Reef.db -> staging file at /data/store-staging.db. Microsoft.Data.Sqlite creates + // the file on first use, so there's no need to touch the filesystem here. + private string ResolveStagingDbPath() + { + var appDbPath = ExtractSqliteDataSource(_connectionString); + var directory = Path.GetDirectoryName(Path.GetFullPath(appDbPath)); + return string.IsNullOrEmpty(directory) + ? StagingDbFileName + : Path.Combine(directory, StagingDbFileName); + } + + private static string ExtractSqliteDataSource(string connectionString) + { + // Reef's own connection string is plain SQLite "Data Source=..." (e.g. "Data Source=Reef.db"), + // never a hybrid-encrypted user Connection string - safe to parse with the builder directly. + try + { + return new SqliteConnectionStringBuilder(connectionString).DataSource; + } + catch + { + return "Reef.db"; + } + } + + // Find-or-create by name "{Recipe.Name} (Store)" - one Group per recipe, reused across + // every run of that recipe rather than spawned anew each time. + private async Task FindOrCreateRecipeGroupAsync(RecipeDefinition recipe, CancellationToken ct) + { + var name = $"{recipe.Name} (Store)"; + var existing = await _groupService.GetByNameAsync(name); + if (existing is not null) + return existing.Id; + + var newGroup = new ProfileGroup + { + Name = name, + Description = $"Created automatically by the Store {recipe.Name} recipe." + }; + + var id = await _groupService.CreateAsync(newGroup, "store-wizard"); + Log.Information("Auto-provisioned Group '{Name}' (ID: {Id}) for recipe '{RecipeKey}'", name, id, recipe.Key); + return id; + } + + public async Task GetRunStateAsync(int runId, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var stepState = DeserializeStepState(run.StepStateJson); + + return new RecipeRunStateDto + { + RunId = run.Id, + RecipeKey = run.RecipeKey, + RecipeName = recipe.Name, + Status = run.Status, + CurrentStepKey = run.CurrentStepKey, + CreatedAt = run.CreatedAt, + UpdatedAt = run.UpdatedAt, + CompletedAt = run.CompletedAt, + Steps = recipe.Steps.Select(step => ToDto(step, stepState.GetValueOrDefault(step.StepKey) ?? new RecipeStepState(), recipe.Key)).ToList() + }; + } + + public async Task CompleteRunAsync(int runId, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var stepState = DeserializeStepState(run.StepStateJson); + var blockingStep = recipe.Steps.FirstOrDefault(s => + !s.IsOptional && s.VerifierKind is not null && + stepState.GetValueOrDefault(s.StepKey)?.Verified != true); + + if (blockingStep is not null) + throw new InvalidOperationException($"Step '{blockingStep.Title}' must be verified before completing the recipe."); + + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + var now = DateTime.UtcNow; + await conn.ExecuteAsync(new CommandDefinition( + "UPDATE RecipeRuns SET Status = 'Completed', UpdatedAt = @Now, CompletedAt = @Now WHERE Id = @Id", + new { Id = runId, Now = now }, cancellationToken: ct)); + + Log.Information("Recipe run completed: {RunId} ({RecipeKey})", runId, run.RecipeKey); + } + + public async Task AbandonRunAsync(int runId, CancellationToken ct = default) + { + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + await conn.ExecuteAsync(new CommandDefinition( + "UPDATE RecipeRuns SET Status = 'Abandoned', UpdatedAt = @Now WHERE Id = @Id", + new { Id = runId, Now = DateTime.UtcNow }, cancellationToken: ct)); + + Log.Information("Recipe run abandoned: {RunId}", runId); + } + + public async Task ExecuteStepAsync(int runId, string stepKey, Dictionary stepParams, int? userId, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var step = recipe.Steps.FirstOrDefault(s => s.StepKey == stepKey) + ?? throw new InvalidOperationException($"Step '{stepKey}' is not part of recipe '{run.RecipeKey}'."); + + var stepState = DeserializeStepState(run.StepStateJson); + var existing = stepState.GetValueOrDefault(stepKey); + + int? entityId = step.EntityType switch + { + RecipeEntityType.Connection => await SaveConnectionAsync(existing, stepParams, userId, ct), + RecipeEntityType.Group => await SaveGroupAsync(existing, stepParams, ct), + RecipeEntityType.Destination => await SaveDestinationAsync(existing, stepParams, stepState, ct), + RecipeEntityType.StagingTable => await CreateStagingTableAsync(recipe.Key, stepKey, stepParams, ct), + RecipeEntityType.ImportProfile => await SaveImportProfileAsync(recipe.Key, stepKey, existing, stepParams, userId, ct), + RecipeEntityType.QueryTemplate => await SaveQueryTemplateAsync(recipe.Key, stepKey, existing, stepParams, ct), + RecipeEntityType.Profile => await SaveProfileAsync(recipe.Key, stepKey, existing, stepParams, userId, ct), + RecipeEntityType.Job => await SaveJobAsync(stepParams, ct), + _ => existing?.EntityId + }; + + var newState = new RecipeStepState + { + EntityId = entityId, + // No verifier (Group, optional Jobs) means nothing left to live-check; steps + // with a verifier require a fresh VerifyStepAsync call after every save. + Verified = step.VerifierKind is null, + LastVerifiedAt = step.VerifierKind is null ? existing?.LastVerifiedAt : null, + LastVerifyMessage = step.VerifierKind is null ? existing?.LastVerifyMessage : null, + Params = stepParams, + Skipped = false + }; + + stepState[stepKey] = newState; + await PersistStepStateAsync(runId, stepKey, recipe, stepState, ct); + + return ToDto(step, newState, recipe.Key); + } + + public async Task SkipStepAsync(int runId, string stepKey, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var step = recipe.Steps.FirstOrDefault(s => s.StepKey == stepKey) + ?? throw new InvalidOperationException($"Step '{stepKey}' is not part of recipe '{run.RecipeKey}'."); + + if (!step.IsOptional) + throw new InvalidOperationException($"Step '{step.Title}' is required and cannot be skipped."); + + return await MarkStepSkippedAsync(runId, run, recipe, step, ct); + } + + // Shared by both the per-step "Skip this step" button (optional steps only) and + // SkipFlowGroupAsync's bulk skip of a whole declined flow - the actual state mutation + // (write Skipped=true, persist) never duplicates, only the eligibility check above differs. + private async Task MarkStepSkippedAsync(int runId, RecipeRun run, RecipeDefinition recipe, RecipeStepDefinition step, CancellationToken ct) + { + var stepState = DeserializeStepState(run.StepStateJson); + var newState = new RecipeStepState { Skipped = true }; + stepState[step.StepKey] = newState; + await PersistStepStateAsync(runId, step.StepKey, recipe, stepState, ct); + + return ToDto(step, newState, recipe.Key); + } + + // Generic flow-group skip: any recipe with >1 distinct FlowGroup among its non-shared + // steps is eligible (checked by callers via HasMultipleFlowGroups), so this never needs + // to know which recipe it's skipping for. IsShared steps are never touched - they're + // skipped via FlowGroup filtering (shared steps have no FlowGroup) rather than a hardcoded + // exclusion. Required (non-optional) steps in a declined flow are skipped too, since + // "the user opted out of this entire flow" is a different, deliberate decision than the + // per-step "Skip this step" affordance IsOptional gates. + public async Task> SkipFlowGroupAsync(int runId, string flowGroup, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var steps = recipe.Steps.Where(s => !s.IsShared && s.FlowGroup == flowGroup).ToList(); + if (steps.Count == 0) + throw new InvalidOperationException($"Flow group '{flowGroup}' is not part of recipe '{run.RecipeKey}'."); + + var results = new List(); + foreach (var step in steps) + { + // Re-fetch so each iteration sees the previous iteration's persisted state. + run = await GetRunOrThrowAsync(runId, ct); + results.Add(await MarkStepSkippedAsync(runId, run, recipe, step, ct)); + } + + Log.Information("Recipe run {RunId} skipped flow group '{FlowGroup}' ({Count} steps)", runId, flowGroup, steps.Count); + return results; + } + + public static bool HasMultipleFlowGroups(RecipeDefinition recipe) => + recipe.Steps.Where(s => !s.IsShared).Select(s => s.FlowGroup).Distinct().Count() > 1; + + public async Task VerifyStepAsync(int runId, string stepKey, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var recipe = RecipeRegistry.GetByKey(run.RecipeKey) + ?? throw new InvalidOperationException($"Recipe '{run.RecipeKey}' no longer exists."); + + var step = recipe.Steps.FirstOrDefault(s => s.StepKey == stepKey) + ?? throw new InvalidOperationException($"Step '{stepKey}' is not part of recipe '{run.RecipeKey}'."); + + if (step.VerifierKind is not { } verifierKind) + throw new InvalidOperationException($"Step '{step.Title}' has no live verification."); + + var stepState = DeserializeStepState(run.StepStateJson); + var current = stepState.GetValueOrDefault(stepKey) + ?? throw new InvalidOperationException($"Step '{step.Title}' has not been saved yet."); + + var verifyContext = BuildVerifyContext(recipe.Key, step, current, stepState); + var verifier = _verifiers[verifierKind]; + var result = await verifier.VerifyAsync(verifyContext, ct); + + current.Verified = result.Success; + current.LastVerifiedAt = DateTime.UtcNow; + current.LastVerifyMessage = result.Success ? result.Message : $"{result.Message}"; + stepState[stepKey] = current; + + await PersistStepStateAsync(runId, stepKey, recipe, stepState, ct, advanceOnSuccess: result.Success); + + Log.Information("Recipe run {RunId} step {StepKey} verified={Success}: {Message}", runId, stepKey, result.Success, result.Message); + + return ToDto(step, current, recipe.Key); + } + + private RecipeVerifyContext BuildVerifyContext(string recipeKey, RecipeStepDefinition step, RecipeStepState current, Dictionary allSteps) + { + var connectionId = allSteps.GetValueOrDefault("connection")?.EntityId; + + // Falls back to the recipe's own staging-table step's saved name, never a hardcoded + // default - otherwise a recipe with no staging table (ErrorDigestRecipe) would get + // defaulted to "StoreOrders" just because it reuses the same step keys. Plain step + // keys are shared by WooCommerceRecipe, WooCommerceTrackingRecipe, and MagentoRecipe - + // each is a single-flow (or now-single-flow) recipe, so no "shipments-" prefix needed. + var tableName = step.StepKey switch + { + "staging-table" => current.Params.GetValueOrDefault("tableName") as string, + "query-template" => allSteps.GetValueOrDefault("staging-table")?.Params.GetValueOrDefault("tableName") as string, + "export-profile" => allSteps.GetValueOrDefault("staging-table")?.Params.GetValueOrDefault("tableName") as string, + _ => null + }; + + var mockTemplateRow = recipeKey.Equals(ErrorDigestRecipe.Key, StringComparison.OrdinalIgnoreCase) + ? ErrorDigestRecipe.MockDigestRow() + : recipeKey.Equals(ExactGlobeRecipe.Key, StringComparison.OrdinalIgnoreCase) + ? (IsItemsStep(step.StepKey) ? ExactGlobeRecipe.MockItemRow() : ExactGlobeRecipe.MockDebtorRow()) + : recipeKey.Equals(UblExportRecipe.Key, StringComparison.OrdinalIgnoreCase) + ? (IsOrderStep(step.StepKey) ? UblExportRecipe.MockOrderRow() + : IsDespatchStep(step.StepKey) ? UblExportRecipe.MockDespatchAdviceRow() + : IsInventoryStep(step.StepKey) ? UblExportRecipe.MockInventoryRow() + : UblExportRecipe.MockInvoiceRow()) + : null; + + return new RecipeVerifyContext + { + EntityId = current.EntityId, + ConnectionId = connectionId, + TableName = tableName, + Params = current.Params, + MockTemplateRow = mockTemplateRow + }; + } + + private async Task SaveConnectionAsync(RecipeStepState? existing, Dictionary p, int? userId, CancellationToken ct) + { + var name = RequireString(p, "name"); + var type = RequireString(p, "type"); + var connectionString = RequireString(p, "connectionString"); + + if (existing?.EntityId is { } id) + { + var conn = await _connectionService.GetByIdAsync(id, ct) ?? throw new InvalidOperationException("Connection no longer exists."); + conn.Name = name; + conn.Type = type; + conn.ConnectionString = connectionString; + await _connectionService.UpdateAsync(conn, ct); + return id; + } + + var newConn = new Connection { Name = name, Type = type, ConnectionString = connectionString, Tags = "", Hash = "" }; + return await _connectionService.CreateAsync(newConn, userId, ct); + } + + private async Task SaveGroupAsync(RecipeStepState? existing, Dictionary p, CancellationToken ct) + { + var name = RequireString(p, "name"); + var description = p.GetValueOrDefault("description") as string ?? ""; + + if (existing?.EntityId is { } id) + { + var group = await _groupService.GetByIdAsync(id) ?? throw new InvalidOperationException("Group no longer exists."); + group.Name = name; + group.Description = description; + await _groupService.UpdateAsync(group); + return id; + } + + var newGroup = new ProfileGroup { Name = name, Description = description }; + return await _groupService.CreateAsync(newGroup, "store-wizard"); + } + + private async Task SaveDestinationAsync(RecipeStepState? existing, Dictionary p, Dictionary allSteps, CancellationToken ct) + { + if (p.GetValueOrDefault("reuseExistingDestinationId") is { } reuseRaw && TryToInt(reuseRaw, out var reuseId)) + { + var reused = await _destinationService.GetByIdForExecutionAsync(reuseId, ct) + ?? throw new InvalidOperationException($"Destination {reuseId} not found."); + return reused.Id; + } + + var name = RequireString(p, "name"); + var configurationJson = RequireString(p, "configurationJson"); + + if (existing?.EntityId is { } id) + { + var dest = await _destinationService.GetByIdForExecutionAsync(id, ct) ?? throw new InvalidOperationException("Destination no longer exists."); + dest.Name = name; + dest.ConfigurationJson = configurationJson; + await _destinationService.UpdateAsync(dest, ct); + return id; + } + + var newDest = new Destination { Name = name, Type = DestinationType.Email, ConfigurationJson = configurationJson }; + var created = await _destinationService.CreateAsync(newDest, ct); + return created.Id; + } + + // "Did this run already create a Destination?" - no cross-run search, just this run's + // shared StepStateJson["destination"] entry. + public async Task GetReusableDestinationIdAsync(int runId, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var stepState = DeserializeStepState(run.StepStateJson); + var destStep = stepState.GetValueOrDefault("destination"); + return destStep?.EntityId; + } + + private static bool TryToInt(object? value, out int result) + { + switch (value) + { + case int i: result = i; return true; + case JsonElement je when je.TryGetInt32(out var i): result = i; return true; + case string s when int.TryParse(s, out var i): result = i; return true; + default: result = 0; return false; + } + } + + // WooCommerceTrackingRecipe is now its own RecipeDefinition (split out of WooCommerceRecipe's + // former Flow B) - dispatch on recipe.Key, the same way MagentoRecipe is told apart from + // WooCommerceRecipe, rather than a "shipments-" step-key prefix that no longer exists. + private static bool IsWooCommerceTrackingRecipe(string recipeKey) => + recipeKey.Equals(WooCommerceTrackingRecipe.Key, StringComparison.OrdinalIgnoreCase); + + // ExactGlobeRecipe's two flows (Debtors/Items) are a different two-flow shape than + // WooCommerce's Order Confirmation/Tracking Link - same "(recipeKey, topic)" dispatch + // mechanism, distinct helper since "shipments" semantics don't apply here. + private static bool IsDebtorsStep(string stepKey) => stepKey.StartsWith("debtors-", StringComparison.OrdinalIgnoreCase); + private static bool IsItemsStep(string stepKey) => stepKey.StartsWith("items-", StringComparison.OrdinalIgnoreCase); + + // UblExportRecipe's four flows (Invoice/Order/Despatch Advice/Inventory) - same + // "(recipeKey, topic)" dispatch mechanism as ExactGlobeRecipe's Debtors/Items pair, just + // four prefixes instead of two. + private static bool IsInvoiceStep(string stepKey) => stepKey.StartsWith("invoice-", StringComparison.OrdinalIgnoreCase); + private static bool IsOrderStep(string stepKey) => stepKey.StartsWith("order-", StringComparison.OrdinalIgnoreCase); + private static bool IsDespatchStep(string stepKey) => stepKey.StartsWith("despatch-", StringComparison.OrdinalIgnoreCase); + private static bool IsInventoryStep(string stepKey) => stepKey.StartsWith("inventory-", StringComparison.OrdinalIgnoreCase); + + private async Task CreateStagingTableAsync(string recipeKey, string stepKey, Dictionary p, CancellationToken ct) + { + var isTracking = IsWooCommerceTrackingRecipe(recipeKey); + var isMagento = recipeKey.Equals(MagentoRecipe.Key, StringComparison.OrdinalIgnoreCase); + var defaultTableName = isMagento + ? MagentoRecipe.StagingTableName + : isTracking ? WooCommerceRecipe.ShipmentsStagingTableName : WooCommerceRecipe.StagingTableName; + + var connectionId = RequireInt(p, "connectionId"); + var tableName = p.GetValueOrDefault("tableName") as string ?? defaultTableName; + + var connection = await _connectionService.GetByIdAsync(connectionId, ct) + ?? throw new InvalidOperationException($"Connection {connectionId} not found."); + + var exists = await TableExistsAsync(connection, tableName, ct); + if (!exists) + { + var baseDdl = isMagento + ? MagentoRecipe.GetStagingTableDdl(connection.Type) + : isTracking + ? WooCommerceRecipe.GetShipmentsStagingTableDdl(connection.Type) + : WooCommerceRecipe.GetStagingTableDdl(connection.Type); + var ddl = baseDdl.Replace(defaultTableName, tableName); + await ExecuteDdlAsync(connection, ddl, ct); + Log.Information("Recipe wizard created staging table {Table} on connection {ConnectionId}", tableName, connectionId); + } + + return connectionId; + } + + private async Task SaveImportProfileAsync(string recipeKey, string stepKey, RecipeStepState? existing, Dictionary p, int? userId, CancellationToken ct) + { + var profile = BuildImportProfileFromParams(recipeKey, stepKey, p); + + if (existing?.EntityId is { } id) + { + profile.Id = id; + await _importProfileService.UpdateAsync(profile); + return id; + } + + return await _importProfileService.CreateAsync(profile, userId); + } + + private static ImportProfile BuildImportProfileFromParams(string recipeKey, string stepKey, Dictionary p) + { + var isMagento = recipeKey.Equals(MagentoRecipe.Key, StringComparison.OrdinalIgnoreCase); + var defaultTableName = isMagento + ? MagentoRecipe.StagingTableName + : IsWooCommerceTrackingRecipe(recipeKey) ? WooCommerceRecipe.ShipmentsStagingTableName : WooCommerceRecipe.StagingTableName; + var defaultUpsertKey = isMagento ? "MagentoOrderId" : "WooOrderId"; + return new() + { + Name = RequireString(p, "name"), + GroupId = p.GetValueOrDefault("groupId") is int gi ? gi : (p.GetValueOrDefault("groupId") as int?), + SourceType = "Http", + SourceConfig = p.GetValueOrDefault("sourceConfig") as string, + HttpMethod = p.GetValueOrDefault("httpMethod") as string ?? "GET", + HttpPaginationEnabled = p.GetValueOrDefault("httpPaginationEnabled") is bool b && b, + HttpPaginationConfig = p.GetValueOrDefault("httpPaginationConfig") as string, + HttpDataRootPath = p.GetValueOrDefault("httpDataRootPath") as string, + SourceFormat = "JSON", + TargetType = "Database", + TargetConnectionId = RequireInt(p, "targetConnectionId"), + TargetTable = p.GetValueOrDefault("targetTable") as string ?? defaultTableName, + LoadStrategy = "Upsert", + UpsertKeyColumns = p.GetValueOrDefault("upsertKeyColumns") as string ?? defaultUpsertKey, + ColumnMappingsJson = p.GetValueOrDefault("columnMappingsJson") as string, + Hash = "" + }; + } + + private static (string Template, string Description, string Tags, string OutputFormat) GetQueryTemplateDefaults(string recipeKey, string stepKey) + { + var isItems = IsItemsStep(stepKey); + return recipeKey switch + { + ErrorDigestRecipe.Key => (ErrorDigestEmailTemplate.DailyDigestHtml, "Created by the Store System Error Daily Digest recipe", "store,digest", "HTML"), + MagentoRecipe.Key => (MagentoEmailTemplates.TrackingUpdateHtml, "Created by the Store Magento Tracking Link recipe", "store,magento", "HTML"), + WooCommerceTrackingRecipe.Key => (WooCommerceEmailTemplates.TrackingUpdateHtml, "Created by the Store WooCommerce Tracking Link recipe", "store,woocommerce", "HTML"), + ExactGlobeRecipe.Key when isItems => (ExactGlobeTemplates.ItemsXmlTemplate, "Created by the Store Exact Globe+ Items Export recipe", "store,exact-globe", "XML"), + ExactGlobeRecipe.Key => (ExactGlobeTemplates.DebtorsXmlTemplate, "Created by the Store Exact Globe+ Debtors Export recipe", "store,exact-globe", "XML"), + UblExportRecipe.Key when IsOrderStep(stepKey) => (UblExportTemplates.OrderXmlTemplate, "Created by the Store UBL Standard Order Export recipe", "store,ubl", "XML"), + UblExportRecipe.Key when IsDespatchStep(stepKey) => (UblExportTemplates.DespatchAdviceXmlTemplate, "Created by the Store UBL Standard Despatch Advice Export recipe", "store,ubl", "XML"), + UblExportRecipe.Key when IsInventoryStep(stepKey) => (UblExportTemplates.InventoryXmlTemplate, "Created by the Store UBL Standard Inventory Export recipe", "store,ubl", "XML"), + UblExportRecipe.Key => (UblExportTemplates.InvoiceXmlTemplate, "Created by the Store UBL Standard Invoice Export recipe", "store,ubl", "XML"), + _ => (WooCommerceEmailTemplates.OrderConfirmationHtml, "Created by the Store WooCommerce Order Confirmation recipe", "store,woocommerce", "HTML") + }; + } + + private async Task SaveQueryTemplateAsync(string recipeKey, string stepKey, RecipeStepState? existing, Dictionary p, CancellationToken ct) + { + var name = RequireString(p, "name"); + var (defaultTemplate, description, tags, outputFormat) = GetQueryTemplateDefaults(recipeKey, stepKey); + var template = p.GetValueOrDefault("template") as string ?? defaultTemplate; + + if (existing?.EntityId is { } id) + { + var existingTemplate = await _queryTemplateService.GetByIdAsync(id) ?? throw new InvalidOperationException("QueryTemplate no longer exists."); + existingTemplate.Name = name; + existingTemplate.Template = template; + await _queryTemplateService.UpdateAsync(existingTemplate); + return id; + } + + var newTemplate = new QueryTemplate + { + Name = name, + Description = description, + Type = QueryTemplateType.ScribanTemplate, + Template = template, + OutputFormat = outputFormat, + Tags = tags + }; + var created = await _queryTemplateService.CreateAsync(newTemplate); + return created.Id; + } + + private async Task SaveProfileAsync(string recipeKey, string stepKey, RecipeStepState? existing, Dictionary p, int? userId, CancellationToken ct) + { + var profile = BuildExportProfileFromParams(recipeKey, stepKey, p); + + if (existing?.EntityId is { } id) + { + profile.Id = id; + await _profileService.UpdateAsync(profile, ct); + return id; + } + + return await _profileService.CreateAsync(profile, userId, ct); + } + + private static Profile BuildExportProfileFromParams(string recipeKey, string stepKey, Dictionary p) + { + if (recipeKey.Equals(ExactGlobeRecipe.Key, StringComparison.OrdinalIgnoreCase)) + return BuildExactGlobeExportProfileFromParams(stepKey, p); + + if (recipeKey.Equals(UblExportRecipe.Key, StringComparison.OrdinalIgnoreCase)) + return BuildUblExportProfileFromParams(stepKey, p); + + var isTracking = IsWooCommerceTrackingRecipe(recipeKey); + var isErrorDigest = recipeKey.Equals(ErrorDigestRecipe.Key, StringComparison.OrdinalIgnoreCase); + var isMagento = recipeKey.Equals(MagentoRecipe.Key, StringComparison.OrdinalIgnoreCase); + + var defaultQuery = (isErrorDigest, isMagento, isTracking) switch + { + (true, _, _) => ErrorDigestRecipe.DefaultExportQuery, + (_, true, _) => MagentoRecipe.DefaultExportQuery, + (_, _, true) => WooCommerceRecipe.DefaultShipmentsExportQuery, + (_, _, false) => WooCommerceRecipe.DefaultExportQuery + }; + + var profile = new Profile + { + Name = RequireString(p, "name"), + ConnectionId = RequireInt(p, "connectionId"), + GroupId = p.GetValueOrDefault("groupId") is int gi ? gi : (p.GetValueOrDefault("groupId") as int?), + Query = p.GetValueOrDefault("query") as string ?? defaultQuery, + OutputFormat = "HTML", + OutputDestinationType = "Email", + OutputDestinationId = RequireInt(p, "destinationId"), + IsEmailExport = true, + EmailTemplateId = RequireInt(p, "emailTemplateId"), + Hash = "" + }; + + if (isErrorDigest) + { + profile.EmailRecipientsColumn = p.GetValueOrDefault("emailRecipientsColumn") as string ?? "email"; + profile.EmailSubjectColumn = p.GetValueOrDefault("emailSubjectColumn") as string ?? "subject"; + profile.UseHardcodedSubject = false; + } + else + { + profile.EmailRecipientsColumn = p.GetValueOrDefault("emailRecipientsColumn") as string ?? "CustomerEmail"; + profile.EmailSubjectHardcoded = p.GetValueOrDefault("emailSubject") as string ?? (isTracking || isMagento ? "Your order is on its way" : "Your order is confirmed"); + profile.UseHardcodedSubject = true; + } + + return profile; + } + + // File export, not email: OutputDestinationType="Local" + an inline OutputDestinationConfig + // JSON path (ExecutionService's "Priority 3: fall back to profile's inline configuration") + // writes straight to a configured folder, no separate Destination entity required. + private static Profile BuildExactGlobeExportProfileFromParams(string stepKey, Dictionary p) + { + var isItems = IsItemsStep(stepKey); + var defaultQuery = isItems ? ExactGlobeRecipe.DefaultItemsExportQuery : ExactGlobeRecipe.DefaultDebtorsExportQuery; + var defaultPath = isItems ? "exports/exact-globe/items" : "exports/exact-globe/debtors"; + + var outputPath = p.GetValueOrDefault("outputPath") as string ?? defaultPath; + var destinationConfig = JsonSerializer.Serialize(new { path = outputPath }, JsonOptions); + + return new Profile + { + Name = RequireString(p, "name"), + ConnectionId = RequireInt(p, "connectionId"), + GroupId = p.GetValueOrDefault("groupId") is int gi ? gi : (p.GetValueOrDefault("groupId") as int?), + Query = p.GetValueOrDefault("query") as string ?? defaultQuery, + OutputFormat = "XML", + OutputDestinationType = "Local", + OutputDestinationConfig = destinationConfig, + IsEmailExport = false, + TemplateId = RequireInt(p, "templateId"), + Hash = "" + }; + } + + // Same file-export shape as BuildExactGlobeExportProfileFromParams (no email, inline + // OutputDestinationType="Local"/OutputDestinationConfig) but kept as its own method rather + // than folded into that one - ExactGlobe's is a 2-way isItems branch, UBL needs a 4-way + // branch across Invoice/Order/Despatch Advice/Inventory, and merging them would mean every + // call site juggling both step-key vocabularies at once for no shared benefit. + private static Profile BuildUblExportProfileFromParams(string stepKey, Dictionary p) + { + var (defaultQuery, defaultPathSegment) = (IsOrderStep(stepKey), IsDespatchStep(stepKey), IsInventoryStep(stepKey)) switch + { + (true, _, _) => (UblExportRecipe.DefaultOrderExportQuery, "orders"), + (_, true, _) => (UblExportRecipe.DefaultDespatchAdviceExportQuery, "despatch-advices"), + (_, _, true) => (UblExportRecipe.DefaultInventoryExportQuery, "inventory"), + _ => (UblExportRecipe.DefaultInvoiceExportQuery, "invoices") + }; + var defaultPath = $"exports/ubl-standard-export/{defaultPathSegment}"; + + var outputPath = p.GetValueOrDefault("outputPath") as string ?? defaultPath; + var destinationConfig = JsonSerializer.Serialize(new { path = outputPath }, JsonOptions); + + return new Profile + { + Name = RequireString(p, "name"), + ConnectionId = RequireInt(p, "connectionId"), + GroupId = p.GetValueOrDefault("groupId") is int gi ? gi : (p.GetValueOrDefault("groupId") as int?), + Query = p.GetValueOrDefault("query") as string ?? defaultQuery, + OutputFormat = "XML", + OutputDestinationType = "Local", + OutputDestinationConfig = destinationConfig, + IsEmailExport = false, + TemplateId = RequireInt(p, "templateId"), + Hash = "" + }; + } + + private async Task SaveJobAsync(Dictionary p, CancellationToken ct) + { + int? lastId = null; + var hasImportProfile = p.GetValueOrDefault("importProfileId") is int; + + if (hasImportProfile) + { + var importProfileId = (int)p["importProfileId"]!; + var importJob = new Job + { + Name = RequireString(p, "importJobName"), + Type = JobType.ProfileExecution, + ImportProfileId = importProfileId, + ScheduleType = Models.ScheduleType.Interval, + IntervalMinutes = p.GetValueOrDefault("importIntervalMinutes") is int im ? im : 15 + }; + var created = await _jobService.CreateAsync(importJob); + lastId = created.Id; + + if (p.GetValueOrDefault("profileId") is int profileId) + { + var exportJob = new Job + { + Name = RequireString(p, "exportJobName"), + Type = JobType.ProfileExecution, + ProfileId = profileId, + ScheduleType = Models.ScheduleType.OnDependency, + DependsOnJobIds = created.Id.ToString() + }; + var createdExport = await _jobService.CreateAsync(exportJob); + lastId = createdExport.Id; + } + } + else if (p.GetValueOrDefault("profileId") is int standaloneProfileId) + { + // No import side - schedule the export Profile directly on its own interval. + var exportJob = new Job + { + Name = RequireString(p, "exportJobName"), + Type = JobType.ProfileExecution, + ProfileId = standaloneProfileId, + ScheduleType = Models.ScheduleType.Interval, + IntervalMinutes = p.GetValueOrDefault("importIntervalMinutes") is int sim ? sim : 1440 + }; + var created = await _jobService.CreateAsync(exportJob); + lastId = created.Id; + } + + return lastId; + } + + // Belt-and-suspenders fast path on top of the polling Import Job from SaveJobAsync - + // fast-tracks an immediate re-pull instead of waiting for the interval. + public async Task<(int WebhookId, string Url)> RegisterTrackingWebhookAsync(int runId, string requestScheme, string requestHost, CancellationToken ct = default) + { + var run = await GetRunOrThrowAsync(runId, ct); + var stepState = DeserializeStepState(run.StepStateJson); + var importStep = stepState.GetValueOrDefault("import-profile") + ?? throw new InvalidOperationException("The Tracking Link import step has not been saved yet."); + + var importProfileId = importStep.EntityId + ?? throw new InvalidOperationException("The Tracking Link ImportProfile has not been saved yet."); + + var (webhookId, token) = await _webhookService.CreateWebhookForImportProfileAsync(importProfileId); + var url = $"{requestScheme}://{requestHost}/webhooks/{token}"; + + Log.Information("Recipe run {RunId} registered Tracking Link webhook {WebhookId} for ImportProfile {ImportProfileId}", runId, webhookId, importProfileId); + + return (webhookId, url); + } + + private async Task TableExistsAsync(Connection connection, string tableName, CancellationToken ct) + { + await using var db = OpenTargetConnection(connection); + await db.OpenAsync(ct); + + var sql = connection.Type switch + { + "MySQL" or "MariaDB" => "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @TableName", + "PostgreSQL" => "SELECT COUNT(*) FROM pg_tables WHERE tablename = @TableName", + "Sqlite" => "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name = @TableName", + _ => "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = @TableName" + }; + + var count = await db.ExecuteScalarAsync(new CommandDefinition(sql, new { TableName = tableName }, commandTimeout: 15, cancellationToken: ct)); + return count > 0; + } + + private async Task ExecuteDdlAsync(Connection connection, string ddl, CancellationToken ct) + { + await using var db = OpenTargetConnection(connection); + await db.OpenAsync(ct); + await db.ExecuteAsync(new CommandDefinition(ddl, commandTimeout: 30, cancellationToken: ct)); + } + + private System.Data.Common.DbConnection OpenTargetConnection(Connection connection) + { + var cs = _encryptionService.IsEncrypted(connection.ConnectionString) + ? _encryptionService.Decrypt(connection.ConnectionString) + : connection.ConnectionString; + + return connection.Type switch + { + "SqlServer" => new Microsoft.Data.SqlClient.SqlConnection(cs), + "MySQL" or "MariaDB" => new MySqlConnector.MySqlConnection(cs), + "PostgreSQL" => new Npgsql.NpgsqlConnection(cs), + // The Sqlite Connection type here is the user's chosen staging file (from `cs`), + // never Reef's own app database (`_connectionString`). + "Sqlite" => new SqliteConnection(cs), + _ => new Microsoft.Data.SqlClient.SqlConnection(cs) + }; + } + + private async Task PersistStepStateAsync(int runId, string stepKey, RecipeDefinition recipe, Dictionary stepState, CancellationToken ct, bool advanceOnSuccess = false) + { + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + string? nextCurrentStepKey = null; + if (advanceOnSuccess) + { + var stepIndex = recipe.Steps.FindIndex(s => s.StepKey == stepKey); + var next = recipe.Steps.Skip(stepIndex + 1).FirstOrDefault(); + nextCurrentStepKey = next?.StepKey; + } + + var json = JsonSerializer.Serialize(stepState, JsonOptions); + + if (nextCurrentStepKey is not null) + { + await conn.ExecuteAsync(new CommandDefinition( + "UPDATE RecipeRuns SET StepStateJson = @Json, CurrentStepKey = @NextStep, UpdatedAt = @Now WHERE Id = @Id", + new { Id = runId, Json = json, NextStep = nextCurrentStepKey, Now = DateTime.UtcNow }, cancellationToken: ct)); + } + else + { + await conn.ExecuteAsync(new CommandDefinition( + "UPDATE RecipeRuns SET StepStateJson = @Json, UpdatedAt = @Now WHERE Id = @Id", + new { Id = runId, Json = json, Now = DateTime.UtcNow }, cancellationToken: ct)); + } + } + + private async Task GetRunOrThrowAsync(int runId, CancellationToken ct) + { + await using var conn = new SqliteConnection(_connectionString); + await conn.OpenAsync(ct); + + var run = await conn.QueryFirstOrDefaultAsync( + new CommandDefinition("SELECT * FROM RecipeRuns WHERE Id = @Id", new { Id = runId }, cancellationToken: ct)); + + return run ?? throw new InvalidOperationException($"Recipe run {runId} not found."); + } + + private static Dictionary DeserializeStepState(string? json) + { + if (string.IsNullOrWhiteSpace(json)) return new(); + try + { + return JsonSerializer.Deserialize>(json, JsonOptions) ?? new(); + } + catch + { + return new(); + } + } + + private static readonly HashSet SecretParamKeys = new(StringComparer.OrdinalIgnoreCase) + { + "connectionString", "configurationJson", "smtpPassword", "consumerSecret", "sourceConfig" + }; + + private static RecipeStepStateDto ToDto(RecipeStepDefinition step, RecipeStepState state, string recipeKey) + { + var displayParams = state.Params.Where(kv => !SecretParamKeys.Contains(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value); + + // Show the built-in default in the editor up front instead of leaving it blank with + // just an explanatory note - the user can see and tweak it before saving, rather than + // saving blind and hoping the "leave blank to use the built-in template" note was right. + if (step.EntityType == RecipeEntityType.QueryTemplate && state.EntityId is null && !displayParams.ContainsKey("template")) + { + displayParams["template"] = GetQueryTemplateDefaults(recipeKey, step.StepKey).Template; + } + + return new() + { + StepKey = step.StepKey, + Title = step.Title, + EntityType = step.EntityType.ToString(), + IsOptional = step.IsOptional, + IsShared = step.IsShared, + FlowGroup = step.FlowGroup, + CanAutoProvision = step.CanAutoProvision, + HasVerifier = step.VerifierKind is not null, + EntityId = state.EntityId, + Verified = state.Verified, + Skipped = state.Skipped, + LastVerifiedAt = state.LastVerifiedAt, + LastVerifyMessage = state.LastVerifyMessage, + Params = displayParams + }; + } + + private static string RequireString(Dictionary p, string key) => + p.GetValueOrDefault(key) as string ?? throw new InvalidOperationException($"'{key}' is required."); + + private static int RequireInt(Dictionary p, string key) + { + var value = p.GetValueOrDefault(key); + return value switch + { + int i => i, + JsonElement je when je.TryGetInt32(out var i) => i, + string s when int.TryParse(s, out var i) => i, + _ => throw new InvalidOperationException($"'{key}' is required.") + }; + } +} + +public class RecipeListItem +{ + public required string Key { get; set; } + public required string Name { get; set; } + public required string Description { get; set; } + public required string Category { get; set; } + public required string Icon { get; set; } + public int StepCount { get; set; } + public int? ExistingRunId { get; set; } + public int? LastCompletedRunId { get; set; } + public bool IsInstalled { get; set; } +} + +public class RecipeRunStateDto +{ + public int RunId { get; set; } + public required string RecipeKey { get; set; } + public required string RecipeName { get; set; } + public required string Status { get; set; } + public string? CurrentStepKey { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } + public DateTime? CompletedAt { get; set; } + public List Steps { get; set; } = new(); +} + +public class RecipeStepStateDto +{ + public required string StepKey { get; set; } + public required string Title { get; set; } + public required string EntityType { get; set; } + public bool IsOptional { get; set; } + public bool IsShared { get; set; } + public string? FlowGroup { get; set; } + public bool CanAutoProvision { get; set; } + public bool HasVerifier { get; set; } + public int? EntityId { get; set; } + public bool Verified { get; set; } + public bool Skipped { get; set; } + public DateTime? LastVerifiedAt { get; set; } + public string? LastVerifyMessage { get; set; } + + public Dictionary Params { get; set; } = new(); +} diff --git a/Source/Reef/Core/Targets/DatabaseImportTarget.cs b/Source/Reef/Core/Targets/DatabaseImportTarget.cs index e2335f2..8f7a492 100644 --- a/Source/Reef/Core/Targets/DatabaseImportTarget.cs +++ b/Source/Reef/Core/Targets/DatabaseImportTarget.cs @@ -2,6 +2,7 @@ using System.Data.Common; using System.Text; using Dapper; +using Microsoft.Data.Sqlite; using Microsoft.Data.SqlClient; using MySqlConnector; using Npgsql; @@ -12,7 +13,7 @@ namespace Reef.Core.Targets; /// -/// Writes imported rows to a SQL Server, MySQL, or PostgreSQL database. +/// Writes imported rows to a SQL Server, MySQL, PostgreSQL, or SQLite database. /// Supports Insert, Upsert, FullReplace, and Append load strategies. /// public class DatabaseImportTarget : IImportTarget @@ -152,6 +153,7 @@ public async Task> GetTableSchemaAsync( "SQLSERVER" or "MSSQL" => SqlServerSchemaQuery(schema, table), "MYSQL" or "MARIADB" => MySqlSchemaQuery(table), "POSTGRESQL" or "POSTGRES" => PostgresSchemaQuery(schema, table), + "SQLITE" => SqliteSchemaQuery(table), _ => SqlServerSchemaQuery(schema, table) }; @@ -401,6 +403,7 @@ await db.ExecuteAsync(new CommandDefinition(insertSql, dp, tx, "SQLSERVER" or "MSSQL" => await SqlServerUpsertAsync(db, tx, table, row, keyColumns, context, ct), "MYSQL" or "MARIADB" => await MySqlUpsertAsync(db, tx, table, row, keyColumns, context, ct), "POSTGRESQL" or "POSTGRES" => await PostgresUpsertAsync(db, tx, table, row, keyColumns, context, ct), + "SQLITE" => await SqliteUpsertAsync(db, tx, table, row, keyColumns, context, ct), _ => await SqlServerUpsertAsync(db, tx, table, row, keyColumns, context, ct) }; } @@ -501,6 +504,49 @@ ON CONFLICT ({keyList}) {conflictAction} return xmax == 0 ? (1, 0) : (0, 1); } + private async Task<(int, int)> SqliteUpsertAsync( + IDbConnection db, IDbTransaction tx, string table, + Dictionary row, List keys, + ImportWriteContext context, CancellationToken ct) + { + var allCols = row.Keys.ToList(); + var valueCols = allCols.Except(keys).ToList(); + + var colList = string.Join(", ", allCols.Select(c => $"\"{c}\"")); + var parmList = string.Join(", ", allCols.Select(c => $"@{c}")); + var keyList = string.Join(", ", keys.Select(k => $"\"{k}\"")); + + string conflictAction; + if (valueCols.Any()) + { + conflictAction = "DO UPDATE SET " + string.Join(", ", valueCols.Select(c => $"\"{c}\" = excluded.\"{c}\"")); + } + else + { + conflictAction = "DO NOTHING"; + } + + var dp = new DynamicParameters(); + foreach (var kvp in row) dp.Add($"@{kvp.Key}", kvp.Value ?? DBNull.Value); + + // SQLite has no built-in way to distinguish insert vs update from an + // INSERT...ON CONFLICT statement, so check existence first under the same transaction. + var whereClause = string.Join(" AND ", keys.Select(k => $"\"{k}\" = @{k}")); + var existsSql = $@"SELECT COUNT(*) FROM ""{table}"" WHERE {whereClause}"; + var existedBefore = await db.ExecuteScalarAsync(new CommandDefinition(existsSql, dp, tx, + commandTimeout: context.CommandTimeoutSeconds, cancellationToken: ct)) > 0; + + // SQLite (3.24+, bundled with Microsoft.Data.Sqlite) supports upsert via + // INSERT ... ON CONFLICT ... DO UPDATE. + var sql = $@"INSERT INTO ""{table}"" ({colList}) VALUES ({parmList}) +ON CONFLICT ({keyList}) {conflictAction}"; + + await db.ExecuteAsync(new CommandDefinition(sql, dp, tx, + commandTimeout: context.CommandTimeoutSeconds, cancellationToken: ct)); + + return existedBefore ? (0, 1) : (1, 0); + } + // Schema Queries private static string SqlServerSchemaQuery(string? schema, string table) => $@" @@ -563,6 +609,18 @@ AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum"; + private static string SqliteSchemaQuery(string table) => $@" + SELECT + ti.name AS ColumnName, + ti.type AS DataType, + CASE WHEN ti.""notnull"" = 0 THEN 'YES' ELSE 'NO' END AS IsNullable, + ti.pk AS IsPrimaryKey, + NULL AS MaxLength, + NULL AS Precision, + NULL AS Scale + FROM pragma_table_info('{EscapeString(table)}') ti + ORDER BY ti.cid"; + // Helpers private DbConnection OpenConnection(Connection connection) @@ -576,6 +634,9 @@ private DbConnection OpenConnection(Connection connection) "SQLSERVER" or "MSSQL" => new SqlConnection(cs), "MYSQL" or "MARIADB" => new MySqlConnection(cs), "POSTGRESQL" or "POSTGRES" => new NpgsqlConnection(cs), + // Sqlite target points at a dedicated staging file chosen by the caller - + // never Reef's own app database. + "SQLITE" => new SqliteConnection(cs), _ => new SqlConnection(cs) }; } @@ -674,6 +735,8 @@ private static async Task TableExistsAsync(IDbConnection db, string connTy "MYSQL" or "MARIADB" => $"SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{EscapeString(table)}'", "POSTGRESQL" or "POSTGRES" => $"SELECT COUNT(*) FROM pg_tables WHERE tablename = '{EscapeString(table)}' AND schemaname = '{EscapeString(schema ?? "public")}'", + "SQLITE" => + $"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name = '{EscapeString(table)}'", _ => $"SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '{EscapeString(table)}'" + (!string.IsNullOrWhiteSpace(schema) ? $" AND TABLE_SCHEMA = '{EscapeString(schema)}'" : "") }; @@ -703,6 +766,7 @@ private static void HandleRowError(ImportBatchResult result, int rowNum, string? SqlException se => se.Number is 2601 or 2627, MySqlException me => me.Number is 1062, PostgresException pe => pe.SqlState == "23505", + SqliteException sqe => sqe.SqliteErrorCode is 19 or 1555, // SQLITE_CONSTRAINT, SQLITE_CONSTRAINT_PRIMARYKEY _ => ex.Message.Contains("duplicate", StringComparison.OrdinalIgnoreCase) || ex.Message.Contains("UNIQUE", StringComparison.OrdinalIgnoreCase) || ex.Message.Contains("PRIMARY KEY", StringComparison.OrdinalIgnoreCase) @@ -713,8 +777,10 @@ private static void HandleRowError(ImportBatchResult result, int rowNum, string? SqlException se when se.Number is 2601 or 2627 => "Constraint", MySqlException me when me.Number == 1062 => "Constraint", PostgresException pe when pe.SqlState == "23505" => "Constraint", + SqliteException sqe when sqe.SqliteErrorCode is 19 or 1555 => "Constraint", SqlException se when se.Number == -2 => "Timeout", MySqlException me when me.IsTransient => "Timeout", + SqliteException sqe when sqe.SqliteErrorCode == 5 => "Timeout", // SQLITE_BUSY _ when ex.Message.Contains("timeout", StringComparison.OrdinalIgnoreCase) => "Timeout", _ when ex.Message.Contains("type", StringComparison.OrdinalIgnoreCase) => "Type", _ => "Unknown" @@ -724,6 +790,7 @@ _ when ex.Message.Contains("type", StringComparison.OrdinalIgnoreCase) => "Type" { "MYSQL" or "MARIADB" => $"`{name}`", "POSTGRESQL" or "POSTGRES" => $"\"{name}\"", + "SQLITE" => $"\"{name}\"", _ => $"[{name}]" }; diff --git a/Source/Reef/Helpers/DatabaseInitializer.cs b/Source/Reef/Helpers/DatabaseInitializer.cs index 88a8990..b209e3b 100644 --- a/Source/Reef/Helpers/DatabaseInitializer.cs +++ b/Source/Reef/Helpers/DatabaseInitializer.cs @@ -68,6 +68,9 @@ public async Task InitializeAsync() // Application Startup Tracking await CreateApplicationStartupTableAsync(connection); + // Store guided recipe wizard run state + await CreateRecipeRunsTableAsync(connection); + // Apply schema migrations for existing databases await ApplyMigrationsAsync(connection); @@ -1031,6 +1034,30 @@ UPDATE ApplicationStartup } } + // Store guided recipe wizard run state - tracks progress through a recipe's steps + // so a user can close the wizard mid-setup and resume exactly where they left off. + private async Task CreateRecipeRunsTableAsync(SqliteConnection connection) + { + var sql = @" + CREATE TABLE IF NOT EXISTS RecipeRuns ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + RecipeKey TEXT NOT NULL, + Status TEXT NOT NULL DEFAULT 'InProgress', + CurrentStepKey TEXT NULL, + StepStateJson TEXT NULL, + CreatedAt TEXT NOT NULL, + UpdatedAt TEXT NOT NULL, + CreatedBy INTEGER NULL, + CompletedAt TEXT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_reciperuns_recipekey ON RecipeRuns(RecipeKey); + CREATE INDEX IF NOT EXISTS idx_reciperuns_status ON RecipeRuns(Status); + CREATE INDEX IF NOT EXISTS idx_reciperuns_createdby ON RecipeRuns(CreatedBy); + "; + await connection.ExecuteAsync(sql); + } + private async Task CreateDefaultUser(SqliteConnection connection) { const string checkUserSql = "SELECT COUNT(*) FROM Users"; @@ -1509,6 +1536,14 @@ await connection.ExecuteAsync( // Add OutputTargetsJson column to ImportProfileExecutions for fan-out results await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "OutputTargetsJson", "TEXT NULL"); + // Add Script pre/post-process observability columns to ImportProfileExecutions + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PreProcessStdout", "TEXT NULL"); + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PreProcessStderr", "TEXT NULL"); + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PreProcessExitCode", "INTEGER NULL"); + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PostProcessStdout", "TEXT NULL"); + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PostProcessStderr", "TEXT NULL"); + await AddColumnIfNotExistsAsync(connection, "ImportProfileExecutions", "PostProcessExitCode", "INTEGER NULL"); + Log.Debug("Database schema migrations completed"); } @@ -3250,6 +3285,14 @@ StartedAt TEXT NOT NULL DEFAULT (datetime('now')), -- Fan-out output target results (JSON array) OutputTargetsJson TEXT NULL, + -- Script pre/post-process observability + PreProcessStdout TEXT NULL, + PreProcessStderr TEXT NULL, + PreProcessExitCode INTEGER NULL, + PostProcessStdout TEXT NULL, + PostProcessStderr TEXT NULL, + PostProcessExitCode INTEGER NULL, + FOREIGN KEY (ImportProfileId) REFERENCES ImportProfiles(Id) ON DELETE CASCADE ); diff --git a/Source/Reef/Program.cs b/Source/Reef/Program.cs index 44ad4a4..a5b8d17 100644 --- a/Source/Reef/Program.cs +++ b/Source/Reef/Program.cs @@ -22,8 +22,14 @@ namespace Reef; /// /// Main entry point /// -public class Program +public partial class Program { + [GeneratedRegex(@"(.*?)")] + private static partial Regex TitleRegex(); + + [GeneratedRegex(@"(.*?)", RegexOptions.Singleline)] + private static partial Regex ScriptsRegex(); + public static async Task Main(string[] args) { // Register Dapper type handlers for TimeSpan (must be done before any database operations) @@ -219,6 +225,14 @@ public static async Task Main(string[] args) var mfaMigration = new MfaMigration(connectionString); await mfaMigration.ApplyAsync(); + // Run Scripting migration to add stdout/stderr/exit-code columns for Script processing steps + Log.Debug("Running Scripting database migration..."); + var scriptingMigration = new ScriptingMigration(connectionString); + await scriptingMigration.ApplyAsync(); + + var scriptingStats = await scriptingMigration.GetStatsAsync(); + Log.Debug("Scripting Migration Stats: {@Stats}", scriptingStats); + // Resolve any corrupted/stuck jobs on startup Log.Debug("Checking for corrupted jobs..."); var jobService = new JobService(new DatabaseConfig { ConnectionString = connectionString }, builder.Configuration); @@ -344,15 +358,26 @@ private static void RegisterServices( services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); + // Store guided recipe wizard + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + // Import Profile services services.AddScoped(); services.AddScoped(); @@ -557,6 +582,7 @@ private static async Task ConfigureMiddleware(WebApplication app) "profiles.html", "jobs.html", "groups.html", + "store.html", "executions.html", "email-approvals.html", "documentation.html", @@ -565,7 +591,7 @@ private static async Task ConfigureMiddleware(WebApplication app) }; var navPages = new[] { "dashboard", "connections", "destinations", "templates", - "profiles", "jobs", "groups", "executions", + "profiles", "jobs", "groups", "store", "executions", "email-approvals", "documentation", "admin", "account" }; var layoutPath = Path.Combine(viewsFolder, "_layout.html"); @@ -628,10 +654,10 @@ private static async Task ConfigureMiddleware(WebApplication app) ? char.ToUpper(username[0]) + username[1..] : username.ToUpper(); - var titleMatch = Regex.Match(pageContent, @"(.*?)"); + var titleMatch = TitleRegex().Match(pageContent); var pageTitle = titleMatch.Success ? titleMatch.Groups[1].Value : pageName; - var scriptsMatch = Regex.Match(pageContent, @"(.*?)", RegexOptions.Singleline); + var scriptsMatch = ScriptsRegex().Match(pageContent); var extraScripts = scriptsMatch.Success ? scriptsMatch.Groups[1].Value : ""; var contentStart = pageContent.IndexOf("
@@ -756,7 +783,7 @@ INSERT INTO ApplicationStartup (StartupToken, StartedAt, MachineName, Version) { StartupToken = startupToken, StartedAt = DateTime.UtcNow, - MachineName = Environment.MachineName, + Environment.MachineName, Version = version }); diff --git a/Source/Reef/Reef.csproj b/Source/Reef/Reef.csproj index de8ba28..f6f31ce 100644 --- a/Source/Reef/Reef.csproj +++ b/Source/Reef/Reef.csproj @@ -38,6 +38,7 @@ + diff --git a/Source/Reef/Views/_layout.html b/Source/Reef/Views/_layout.html index b21fc73..e101c74 100644 --- a/Source/Reef/Views/_layout.html +++ b/Source/Reef/Views/_layout.html @@ -85,6 +85,15 @@

Groups + + + + + + + + Store +

@@ -126,14 +135,9 @@

Reference

@@ -166,21 +170,21 @@

- + Account - + Settings - +
@@ -1106,6 +1110,35 @@

Scriban Template Syntax

+ + + -