diff --git a/classes/local/step/connector_curl.php b/classes/local/step/connector_curl.php index 11922e7d..ec43e183 100644 --- a/classes/local/step/connector_curl.php +++ b/classes/local/step/connector_curl.php @@ -16,13 +16,6 @@ namespace tool_dataflows\local\step; -use Symfony\Component\Yaml\Yaml; -use tool_dataflows\helper; - -defined('MOODLE_INTERNAL') || die(); - -require_once($CFG->libdir . '/filelib.php'); - /** * CURL connector step type * @@ -33,318 +26,11 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ class connector_curl extends connector_step { - /** Lowest number for HTTP errors. */ public const HTTP_ERROR = 400; /** @var int Time after which curl request is aborted */ public const DEFAULT_TIMEOUT = 60; - /** - * Returns whether or not the step configured, has a side effect. - * - * For curl connectors, it is considered to have a side effect if the target is - * anywhere outside of the scratch directory, the method is anything other than - * 'get' or 'head', or if the 'has side effects' setting is checked. - * - * @return bool whether or not this step has a side effect - * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) - */ - public function has_side_effect(): bool { - if (isset($this->stepdef)) { - $config = $this->get_variables()->get('config'); - - // Destination is outside of scratch directory. - if (!(empty($config->destination) || helper::path_is_relative($config->destination))) { - return true; - } - - // Request is anything other than 'get' or 'head'. - if (!($config->method == 'get' || $config->method == 'head')) { - return true; - } - - // Side effects setting is checked. - if (!empty($config->sideeffects)) { - return true; - } - - return false; - } - return true; - } - - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'curl' => ['type' => PARAM_TEXT], - 'destination' => ['type' => PARAM_PATH], - 'headers' => ['type' => PARAM_RAW], - 'method' => ['type' => PARAM_TEXT], - 'rawpostdata' => ['type' => PARAM_RAW], - 'sideeffects' => ['type' => PARAM_RAW], - 'timeout' => ['type' => PARAM_INT], - ]; - } - - /** - * A list of outputs and their description if applicable. - * - * These fields can be used as aliases in the custom output mapping - * - * @return array of outputs - */ - public function define_outputs(): array { - return [ - 'dbgcommand' => null, - 'response' => [ - 'result' => get_string('connector_curl:output_response_result', 'tool_dataflows'), - 'info' => [ - 'http_code' => null, - 'connect_time' => null, - 'total_time' => null, - 'size_upload' => null, - ], - 'destination' => get_string('connector_curl:destination', 'tool_dataflows'), - ], - ]; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - $ex = \html_writer::nonempty_tag( - 'pre', - htmlspecialchars(get_string('connector_curl:header_format', 'tool_dataflows') . PHP_EOL) - ); - - $urlarray = []; - $urlarray[] =& $mform->createElement('select', 'config_method', '', [ - 'get' => 'GET', - 'post' => 'POST', - 'head' => 'HEAD', - 'patch' => 'PATCH', - 'put' => 'PUT', - ]); - $urlarray[] =& $mform->createElement('text', 'config_curl', ''); - - $mform->addGroup($urlarray, 'buttonar', get_string('connector_curl:curl', 'tool_dataflows'), [' '], false); - $mform->addRule('buttonar', get_string('required'), 'required', null, 'server'); - - $mform->addElement('textarea', 'config_headers', get_string('connector_curl:headers', 'tool_dataflows'), - ['cols' => 50, 'rows' => 7]); - $mform->addElement('static', 'headers_help', '', get_string('connector_curl:field_headers_help', 'tool_dataflows', $ex)); - - $mform->addElement('textarea' , 'config_rawpostdata', get_string('connector_curl:rawpostdata', 'tool_dataflows'), - ['cols' => 50, 'rows' => 7]); - - $mform->addElement('text', 'config_destination', get_string('connector_curl:destination', 'tool_dataflows')); - $mform->addHelpButton('config_destination', 'connector_curl:destination', 'tool_dataflows'); - $mform->addElement('static', 'config_path_help', '', get_string('path_help', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); - - $mform->addElement('checkbox', 'config_sideeffects', get_string('connector_curl:sideeffects', 'tool_dataflows'), - get_string('yes')); - $mform->addHelpButton('config_sideeffects', 'connector_curl:sideeffects', 'tool_dataflows'); - - $mform->hideIf('config_rawpostdata', 'config_method', 'eq', 'get'); - $mform->disabledIf('config_rawpostdata', 'config_method', 'eq', 'get'); - - $mform->addElement('text', 'config_timeout', get_string('connector_curl:timeout', 'tool_dataflows')); - $mform->addHelpButton('config_timeout', 'connector_curl:timeout', 'tool_dataflows'); - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (empty($config->curl)) { - $errors['config_curl'] = get_string( - 'config_field_missing', - 'tool_dataflows', - get_string('connector_curl:curl', 'tool_dataflows'), - true - ); - } - if (empty($config->rawpostdata) && ($config->method === 'put' || $config->method === 'post')) { - $errors['config_rawpostdata'] = get_string( - 'config_field_missing', - 'tool_dataflows', - get_string('connector_curl:rawpostdata', 'tool_dataflows'), - true - ); - } - return empty($errors) ? true : $errors; - } - - /** - * Perform any extra validation that is required only for runs. - * - * @return true|array Will return true or an array of errors. - */ - public function validate_for_run() { - $config = $this->get_variables()->get('config'); - - if (!empty($config->destination)) { - $error = helper::path_validate($config->destination); - if ($error !== true) { - return ['config_destination' => $error]; - } - } - - if (!empty($config->headers)) { - $headers = helper::extract_http_headers($config->headers); - if (!is_array($headers)) { - return ['config_headers' => get_string('connector_curl:headersnotvalid', 'tool_dataflows')]; - } - } - - return true; - } - - /** - * Executes the step - * - * Performs a curl call according to given parameters. - * - * @param mixed|null $input - * @return mixed - */ - public function execute($input = null) { - // Get variables. - $variables = $this->get_variables(); - $config = $variables->get('config'); - $method = $config->method; - - $this->enginestep->log($config->curl); - - // Construct a bash curl command. - // See https://manpages.org/curl. - $dbgcommand = 'curl -s -X ' . strtoupper($method) . ' ' . $config->curl; - - // Extract timeout. - $timeout = (int) $config->timeout ?: self::DEFAULT_TIMEOUT; - - $dbgcommand .= ' --max-time ' . $timeout; - $options = ['CURLOPT_TIMEOUT' => $timeout]; - - // Extract headers. - $headers = helper::extract_http_headers($config->headers); - if ($headers === false) { - throw new \moodle_exception(get_string('connector_curl:headers_invalid', 'tool_dataflows')); - } - - // Add headers to bash command. Headers with no value are ended with a ';' in accordance with the man page. - if (!empty($headers)) { - foreach ($headers as $name => $value) { - if (trim($value) !== '') { - $header = "$name:$value"; - } else { - $header = "$name;"; - } - $dbgcommand .= ' -H ' . helper::bash_escape($header); - } - } - - // Sets post data. - if (!empty($config->rawpostdata)) { - $options['CURLOPT_POSTFIELDS'] = $config->rawpostdata; - $dbgcommand .= ' --data-raw ' . helper::bash_escape($config->rawpostdata); - } - - // Download response to file provided destination is set. - if (!empty($config->destination)) { - if ($config->destination[0] === '/') { - $config->destination = ltrim($config->destination, '/'); - } - $config->destination = $this->enginestep->engine->resolve_path($config->destination); - $file = fopen($config->destination, 'w'); - $options['CURLOPT_FILE'] = $file; - $dbgcommand .= ' --output ' . helper::bash_escape($config->destination); - } - - // Log the raw curl command. - $this->enginestep->log($dbgcommand); - $variables->set('dbgcommand', $dbgcommand); - - // We do not need to go any further if curl is not going to be called. - if ($this->enginestep->engine->isdryrun && $this->has_side_effect()) { - return true; - } - - if ($method === 'post') { - $options['CURLOPT_POST'] = 1; - } - - if ($method === 'put') { - $options['CURLOPT_PUT'] = 1; - } - - $curl = new \curl(); - - // Provided a header is specified, add header to request. - if (!empty($headers)) { - $this->set_headers($curl, $headers); - } - - // Perform call. - $this->enginestep->log('Performing curl call.'); - $result = $curl->$method($config->curl, $options['CURLOPT_POSTFIELDS'] ?? [], $options); - - if (!empty($file)) { - fclose($file); - } - - $info = $curl->get_info(); - // Stores response to be reusable by other steps. - // TODO : Once set_var api is refactored add response. - $response = $curl->getResponse(); - $httpcode = $info['http_code'] ?? null; - $destination = !empty($config->destination) ? $config->destination : null; - $errno = $curl->get_errno(); - - if (($httpcode >= self::HTTP_ERROR || $errno == CURLE_OPERATION_TIMEDOUT)) { - throw new \moodle_exception($httpcode . ':' . $result); - } - - // TODO: It would be good to define and list any fixed but exposed - // fields which the user can use and map to on the edit page. - $variables->set('response', (object) [ - 'result' => $result, - 'info' => (object) $info, - 'destination' => $destination, - ]); - - return true; - } - - /** - * Sets header to curl instance - * - * Prepares headers to proper format for setHeader method. - * - * @param \curl $curl RESTful cURL object - * @param array $headers headers to sanitize - */ - protected function set_headers(\curl $curl, array $headers) { - foreach ($headers as $key => $value) { - $curlheaders[] = "$key: $value"; - } - $curl->setHeader($curlheaders); - } + use curl_trait; } diff --git a/classes/local/step/curl_trait.php b/classes/local/step/curl_trait.php new file mode 100644 index 00000000..b8d40422 --- /dev/null +++ b/classes/local/step/curl_trait.php @@ -0,0 +1,344 @@ +. + +namespace tool_dataflows\local\step; + +use Symfony\Component\Yaml\Yaml; +use tool_dataflows\helper; + +defined('MOODLE_INTERNAL') || die(); + +require_once($CFG->libdir . '/filelib.php'); + +/** + * CURL trait + * + * @package tool_dataflows + * @author Kevin Pham + * @author Ghaly Marc-Alexandre + * @copyright Catalyst IT, 2022 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait curl_trait { + + /** + * Returns whether or not the step configured, has a side effect. + * + * For curl connectors, it is considered to have a side effect if the target is + * anywhere outside of the scratch directory, the method is anything other than + * 'get' or 'head', or if the 'has side effects' setting is checked. + * + * @return bool whether or not this step has a side effect + * @link https://en.wikipedia.org/wiki/Side_effect_(computer_science) + */ + public function has_side_effect(): bool { + if (isset($this->stepdef)) { + $config = $this->get_variables()->get('config'); + + // Destination is outside of scratch directory. + if (!(empty($config->destination) || helper::path_is_relative($config->destination))) { + return true; + } + + // Request is anything other than 'get' or 'head'. + if (!($config->method == 'get' || $config->method == 'head')) { + return true; + } + + // Side effects setting is checked. + if (!empty($config->sideeffects)) { + return true; + } + + return false; + } + return true; + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'curl' => ['type' => PARAM_TEXT], + 'destination' => ['type' => PARAM_PATH], + 'headers' => ['type' => PARAM_RAW], + 'method' => ['type' => PARAM_TEXT], + 'rawpostdata' => ['type' => PARAM_RAW], + 'sideeffects' => ['type' => PARAM_RAW], + 'timeout' => ['type' => PARAM_INT], + ]; + } + + /** + * A list of outputs and their description if applicable. + * + * These fields can be used as aliases in the custom output mapping + * + * @return array of outputs + */ + public function define_outputs(): array { + return [ + 'dbgcommand' => null, + 'response' => [ + 'result' => get_string('connector_curl:output_response_result', 'tool_dataflows'), + 'info' => [ + 'http_code' => null, + 'connect_time' => null, + 'total_time' => null, + 'size_upload' => null, + ], + 'destination' => get_string('connector_curl:destination', 'tool_dataflows'), + ], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $ex = \html_writer::nonempty_tag( + 'pre', + htmlspecialchars(get_string('connector_curl:header_format', 'tool_dataflows') . PHP_EOL) + ); + + $urlarray = []; + $urlarray[] =& $mform->createElement('select', 'config_method', '', [ + 'get' => 'GET', + 'post' => 'POST', + 'head' => 'HEAD', + 'patch' => 'PATCH', + 'put' => 'PUT', + ]); + $urlarray[] =& $mform->createElement('text', 'config_curl', ''); + + $mform->addGroup($urlarray, 'buttonar', get_string('connector_curl:curl', 'tool_dataflows'), [' '], false); + $mform->addRule('buttonar', get_string('required'), 'required', null, 'server'); + + $mform->addElement('textarea', 'config_headers', get_string('connector_curl:headers', 'tool_dataflows'), + ['cols' => 50, 'rows' => 7]); + $mform->addElement('static', 'headers_help', '', get_string('connector_curl:field_headers_help', 'tool_dataflows', $ex)); + + $mform->addElement('textarea' , 'config_rawpostdata', get_string('connector_curl:rawpostdata', 'tool_dataflows'), + ['cols' => 50, 'rows' => 7]); + + $mform->addElement('text', 'config_destination', get_string('connector_curl:destination', 'tool_dataflows')); + $mform->addHelpButton('config_destination', 'connector_curl:destination', 'tool_dataflows'); + $mform->addElement('static', 'config_path_help', '', get_string('path_help', 'tool_dataflows'). + \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); + + $mform->addElement('checkbox', 'config_sideeffects', get_string('connector_curl:sideeffects', 'tool_dataflows'), + get_string('yes')); + $mform->addHelpButton('config_sideeffects', 'connector_curl:sideeffects', 'tool_dataflows'); + + $mform->hideIf('config_rawpostdata', 'config_method', 'eq', 'get'); + $mform->disabledIf('config_rawpostdata', 'config_method', 'eq', 'get'); + + $mform->addElement('text', 'config_timeout', get_string('connector_curl:timeout', 'tool_dataflows')); + $mform->addHelpButton('config_timeout', 'connector_curl:timeout', 'tool_dataflows'); + } + + /** + * Validate the configuration settings. + * + * @param object $config + * @return true|\lang_string[] true if valid, an array of errors otherwise + */ + public function validate_config($config) { + $errors = []; + if (empty($config->curl)) { + $errors['config_curl'] = get_string( + 'config_field_missing', + 'tool_dataflows', + get_string('connector_curl:curl', 'tool_dataflows'), + true + ); + } + if (empty($config->rawpostdata) && ($config->method === 'put' || $config->method === 'post')) { + $errors['config_rawpostdata'] = get_string( + 'config_field_missing', + 'tool_dataflows', + get_string('connector_curl:rawpostdata', 'tool_dataflows'), + true + ); + } + return empty($errors) ? true : $errors; + } + + /** + * Perform any extra validation that is required only for runs. + * + * @return true|array Will return true or an array of errors. + */ + public function validate_for_run() { + $config = $this->get_variables()->get('config'); + + if (!empty($config->destination)) { + $error = helper::path_validate($config->destination); + if ($error !== true) { + return ['config_destination' => $error]; + } + } + + if (!empty($config->headers)) { + $headers = helper::extract_http_headers($config->headers); + if (!is_array($headers)) { + return ['config_headers' => get_string('connector_curl:headersnotvalid', 'tool_dataflows')]; + } + } + + return true; + } + + /** + * Executes the step + * + * Performs a curl call according to given parameters. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + // Get variables. + $variables = $this->get_variables(); + $config = $variables->get('config'); + $method = $config->method; + + $this->enginestep->log($config->curl); + + // Construct a bash curl command. + // See https://manpages.org/curl. + $dbgcommand = 'curl -s -X ' . strtoupper($method) . ' ' . $config->curl; + + // Extract timeout. + $timeout = (int) $config->timeout ?: self::DEFAULT_TIMEOUT; + + $dbgcommand .= ' --max-time ' . $timeout; + $options = ['CURLOPT_TIMEOUT' => $timeout]; + + // Extract headers. + $headers = helper::extract_http_headers($config->headers); + if ($headers === false) { + throw new \moodle_exception(get_string('connector_curl:headers_invalid', 'tool_dataflows')); + } + + // Add headers to bash command. Headers with no value are ended with a ';' in accordance with the man page. + if (!empty($headers)) { + foreach ($headers as $name => $value) { + if (trim($value) !== '') { + $header = "$name:$value"; + } else { + $header = "$name;"; + } + $dbgcommand .= ' -H ' . helper::bash_escape($header); + } + } + + // Sets post data. + if (!empty($config->rawpostdata)) { + $options['CURLOPT_POSTFIELDS'] = $config->rawpostdata; + $dbgcommand .= ' --data-raw ' . helper::bash_escape($config->rawpostdata); + } + + // Download response to file provided destination is set. + if (!empty($config->destination)) { + if ($config->destination[0] === '/') { + $config->destination = ltrim($config->destination, '/'); + } + $config->destination = $this->enginestep->engine->resolve_path($config->destination); + $file = fopen($config->destination, 'w'); + $options['CURLOPT_FILE'] = $file; + $dbgcommand .= ' --output ' . helper::bash_escape($config->destination); + } + + // Log the raw curl command. + $this->enginestep->log($dbgcommand); + $variables->set('dbgcommand', $dbgcommand); + + // We do not need to go any further if curl is not going to be called. + if ($this->enginestep->engine->isdryrun && $this->has_side_effect()) { + return true; + } + + if ($method === 'post') { + $options['CURLOPT_POST'] = 1; + } + + if ($method === 'put') { + $options['CURLOPT_PUT'] = 1; + } + + $curl = new \curl(); + + // Provided a header is specified, add header to request. + if (!empty($headers)) { + $this->set_headers($curl, $headers); + } + + // Perform call. + $this->enginestep->log('Performing curl call.'); + $result = $curl->$method($config->curl, $options['CURLOPT_POSTFIELDS'] ?? [], $options); + + if (!empty($file)) { + fclose($file); + } + + $info = $curl->get_info(); + // Stores response to be reusable by other steps. + // TODO : Once set_var api is refactored add response. + $response = $curl->getResponse(); + $httpcode = $info['http_code'] ?? null; + $destination = !empty($config->destination) ? $config->destination : null; + $errno = $curl->get_errno(); + + if (($httpcode >= self::HTTP_ERROR || $errno == CURLE_OPERATION_TIMEDOUT)) { + throw new \moodle_exception($httpcode . ':' . $result); + } + + // TODO: It would be good to define and list any fixed but exposed + // fields which the user can use and map to on the edit page. + $variables->set('response', (object) [ + 'result' => $result, + 'info' => (object) $info, + 'destination' => $destination, + ]); + + return $input; + } + + /** + * Sets header to curl instance + * + * Prepares headers to proper format for setHeader method. + * + * @param \curl $curl RESTful cURL object + * @param array $headers headers to sanitize + */ + protected function set_headers(\curl $curl, array $headers) { + foreach ($headers as $key => $value) { + $curlheaders[] = "$key: $value"; + } + $curl->setHeader($curlheaders); + } +} diff --git a/classes/local/step/flow_curl.php b/classes/local/step/flow_curl.php new file mode 100644 index 00000000..0f236412 --- /dev/null +++ b/classes/local/step/flow_curl.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * CURL flow step type + * + * @package tool_dataflows + * @author Ghaly Marc-Alexandre + * @copyright Catalyst IT, 2022 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_curl extends flow_step { + /** Lowest number for HTTP errors. */ + public const HTTP_ERROR = 400; + + /** @var int Time after which curl request is aborted */ + public const DEFAULT_TIMEOUT = 60; + + use curl_trait; +} diff --git a/classes/local/step/flow_json.php b/classes/local/step/flow_json.php new file mode 100644 index 00000000..7672b9c0 --- /dev/null +++ b/classes/local/step/flow_json.php @@ -0,0 +1,51 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\local\execution\iterators\dataflow_iterator; +use tool_dataflows\local\execution\iterators\iterator; + +/** + * + * JSON flow step type + * + * @package tool_dataflows + * @author Ghaly Marc-Alexandre + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_json extends flow_step { + /** @var string sort order descending key */ + const DESC = 'desc'; + + /** @var string sort order ascending key */ + const ASC = 'asc'; + + use json_trait; + + /** + * Executes the step + * + * Performs a JSON Parsing. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + return $this->parse_json(); + } +} diff --git a/classes/local/step/json_trait.php b/classes/local/step/json_trait.php new file mode 100644 index 00000000..0fcd82bd --- /dev/null +++ b/classes/local/step/json_trait.php @@ -0,0 +1,207 @@ +. + +namespace tool_dataflows\local\step; + +use html_writer; +use Symfony\Component\ExpressionLanguage\ExpressionLanguage; +use tool_dataflows\helper; +use tool_dataflows\local\execution\iterators\iterator; +use tool_dataflows\local\execution\iterators\dataflow_iterator; + +/** + * JSON trait + * + * @package tool_dataflows + * @author Peter Sistrom + * @author Ghaly Marc-Alexandre + * @copyright 2023, Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait json_trait { + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'pathtojson' => ['type' => PARAM_TEXT], + 'arrayexpression' => ['type' => PARAM_TEXT], + 'arraysortexpression' => ['type' => PARAM_TEXT], + 'sortorder' => ['type' => PARAM_TEXT], + ]; + } + + /** + * Parses json string to php array. + * + * @return mixed + * @throws \moodle_exception + */ + protected function parse_json() { + $config = $this->get_variables()->get('config'); + $jsonstring = $this->get_json_string($config->pathtojson); + + $decodedjson = json_decode($jsonstring); + if (is_null($decodedjson)) { + throw new \moodle_exception(get_string('reader_json:failed_to_decode_json', 'tool_dataflows', $config->pathtojson)); + } + + $arrayexpression = $config->arrayexpression; + $expressionlanguage = new ExpressionLanguage(); + $returnarray = $expressionlanguage->evaluate( + $arrayexpression != '' ? 'data.'.$arrayexpression : 'data', + ['data' => $decodedjson] + ); + + if (is_null($returnarray)) { + throw new \moodle_exception(get_string('reader_json:failed_to_fetch_array', + 'tool_dataflows', $config->arrayexpression)); + } + + $sortbyexpression = $config->arraysortexpression; + + // Sort the parsed array if required. + if ($sortbyexpression !== '') { + return $this->sort_by_config_value($returnarray, $sortbyexpression); + } + + return $returnarray; + } + + /** + * Parses stream to json string. + * + * @param string $path + * @return string json + * @throws \moodle_exception + */ + protected function get_json_string(string $path): string { + $jsonstring = file_get_contents($this->enginestep->engine->resolve_path($path)); + if ($jsonstring === false) { + $this->enginestep->log(error_get_last()['message']); + throw new \moodle_exception(get_string('reader_json:failed_to_open_file', 'tool_dataflows', $path)); + } + + return $jsonstring; + } + + /** + * Sort array by config value. + * + * @param array $array + * @param string $sortbyexpression + */ + public function sort_by_config_value(array $array, string $sortbyexpression): array { + $expressionlanguage = new ExpressionLanguage(); + usort($array, function ($a, $b) use ($sortbyexpression, $expressionlanguage) { + $a = $expressionlanguage->evaluate( + 'data.'.$sortbyexpression, + ['data' => $a] + ); + $b = $expressionlanguage->evaluate( + 'data.'.$sortbyexpression, + ['data' => $b] + ); + return strnatcasecmp($a, $b) * $this->get_sort_order_direction(); + }); + return $array; + } + + /** + * Converts the sort order to an int used to flip the default order + * + * This returns 1 for default sort ASC order and -1 for DESC order + * + * @return int + */ + public function get_sort_order_direction() { + $sortorder = $this->get_variables()->get('config.sortorder'); + if ($sortorder === self::DESC) { + return -1; + } + + return 1; + } + + /** + * Validate the configuration settings. + * + * @param object $config + * @return true|\lang_string[] true if valid, an array of errors otherwise + */ + public function validate_config($config) { + $errors = []; + if (!isset($config->pathtojson)) { + $errors['config_pathtojson'] = get_string('config_field_missing', 'tool_dataflows', 'pathtojson', true); + } else { + $error = helper::path_validate($config->pathtojson); + if ($error !== true) { + $errors['config_pathtojson'] = $error; + } + } + return empty($errors) ? true : $errors; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + // JSON array source. + $mform->addElement('text', 'config_pathtojson', get_string('reader_json:pathtojson', 'tool_dataflows')); + $mform->addElement('static', 'config_json_path_help', '', get_string('path_help', 'tool_dataflows'). + \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); + + // Array iterator value. + $arrayexample = (object) [ + 'data' => (object) [ + 'list' => [ + 'users' => [ + ['id' => '1', 'userdetails' => ['firstname' => 'Bob', 'lastname' => 'Smith', 'name' => 'Name1']], + ], + ], + ], + 'modified' => [1654058940], + 'errors' => [], + ]; + $jsonexample = html_writer::empty_tag('br'). + html_writer::nonempty_tag('pre', json_encode($arrayexample, JSON_PRETTY_PRINT)); + $expression = html_writer::nonempty_tag('code', 'data.list.users'); + + $mform->addElement('text', 'config_arrayexpression', get_string('reader_json:arrayexpression', 'tool_dataflows')); + $mform->addElement('static', 'config_arrayexpression_help', '', + get_string('reader_json:arrayexpression_help', 'tool_dataflows', + ['jsonexample' => $jsonexample, 'expression' => $expression])); + + // JSON array sort by. + $mform->addElement('text', 'config_arraysortexpression', get_string('reader_json:arraysortexpression', 'tool_dataflows')); + $mform->addElement('static', 'config_arraysortexpression_help', '', + get_string('reader_json:arraysortexpression_help', 'tool_dataflows', + html_writer::nonempty_tag('code', 'userdetails.firstname'))); + + // JSON array sort order (ASC, DESC). + $mform->addElement('select', 'config_sortorder', get_string('reader_json:sortorder', 'tool_dataflows'), + ['asc' => get_string(self::ASC), 'desc' => get_string(self::DESC)]); + $mform->addElement('static', 'config_sortorder_help', '', get_string('reader_json:sortorder_help', 'tool_dataflows')); + } +} diff --git a/classes/local/step/reader_json.php b/classes/local/step/reader_json.php index 4d38451d..f4c56510 100644 --- a/classes/local/step/reader_json.php +++ b/classes/local/step/reader_json.php @@ -38,19 +38,7 @@ class reader_json extends reader_step { /** @var string sort order ascending key */ const ASC = 'asc'; - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'pathtojson' => ['type' => PARAM_TEXT], - 'arrayexpression' => ['type' => PARAM_TEXT], - 'arraysortexpression' => ['type' => PARAM_TEXT], - 'sortorder' => ['type' => PARAM_TEXT], - ]; - } + use json_trait; /** * Get the iterator for the step, based on configurations. @@ -61,163 +49,4 @@ public function get_iterator(): iterator { $jsonarray = $this->parse_json(); return new dataflow_iterator($this->enginestep, new \ArrayIterator($jsonarray)); } - - /** - * Parses json string to php array. - * - * @return mixed - * @throws \moodle_exception - */ - protected function parse_json() { - $config = $this->get_variables()->get('config'); - $jsonstring = $this->get_json_string($config->pathtojson); - - $decodedjson = json_decode($jsonstring); - if (is_null($decodedjson)) { - throw new \moodle_exception(get_string('reader_json:failed_to_decode_json', 'tool_dataflows', $config->pathtojson)); - } - - $arrayexpression = $config->arrayexpression; - $expressionlanguage = new ExpressionLanguage(); - $returnarray = $expressionlanguage->evaluate( - $arrayexpression != '' ? 'data.'.$arrayexpression : 'data', - ['data' => $decodedjson] - ); - - if (is_null($returnarray)) { - throw new \moodle_exception(get_string('reader_json:failed_to_fetch_array', - 'tool_dataflows', $config->arrayexpression)); - } - - $sortbyexpression = $config->arraysortexpression; - - // Sort the parsed array if required. - if ($sortbyexpression !== '') { - return $this->sort_by_config_value($returnarray, $sortbyexpression); - } - - return $returnarray; - } - - /** - * Parses stream to json string. - * - * @param string $path - * @return string json - * @throws \moodle_exception - */ - protected function get_json_string(string $path): string { - $jsonstring = file_get_contents($this->enginestep->engine->resolve_path($path)); - if ($jsonstring === false) { - $this->enginestep->log(error_get_last()['message']); - throw new \moodle_exception(get_string('reader_json:failed_to_open_file', 'tool_dataflows', $path)); - } - - return $jsonstring; - } - - /** - * Sort array by config value. - * - * @param array $array - * @param string $sortbyexpression - */ - public function sort_by_config_value(array $array, string $sortbyexpression): array { - $expressionlanguage = new ExpressionLanguage(); - usort($array, function ($a, $b) use ($sortbyexpression, $expressionlanguage) { - $a = $expressionlanguage->evaluate( - 'data.'.$sortbyexpression, - ['data' => $a] - ); - $b = $expressionlanguage->evaluate( - 'data.'.$sortbyexpression, - ['data' => $b] - ); - return strnatcasecmp($a, $b) * $this->get_sort_order_direction(); - }); - return $array; - } - - /** - * Converts the sort order to an int used to flip the default order - * - * This returns 1 for default sort ASC order and -1 for DESC order - * - * @return int - */ - public function get_sort_order_direction() { - $sortorder = $this->get_variables()->get('config.sortorder'); - if ($sortorder === self::DESC) { - return -1; - } - - return 1; - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (!isset($config->pathtojson)) { - $errors['config_pathtojson'] = get_string('config_field_missing', 'tool_dataflows', 'pathtojson', true); - } else { - $error = helper::path_validate($config->pathtojson); - if ($error !== true) { - $errors['config_pathtojson'] = $error; - } - } - return empty($errors) ? true : $errors; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - // JSON array source. - $mform->addElement('text', 'config_pathtojson', get_string('reader_json:pathtojson', 'tool_dataflows')); - $mform->addElement('static', 'config_json_path_help', '', get_string('path_help', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('path_help_examples', 'tool_dataflows'))); - - // Array iterator value. - $arrayexample = (object) [ - 'data' => (object) [ - 'list' => [ - 'users' => [ - ['id' => '1', 'userdetails' => ['firstname' => 'Bob', 'lastname' => 'Smith', 'name' => 'Name1']], - ], - ], - ], - 'modified' => [1654058940], - 'errors' => [], - ]; - $jsonexample = html_writer::empty_tag('br'). - html_writer::nonempty_tag('pre', json_encode($arrayexample, JSON_PRETTY_PRINT)); - $expression = html_writer::nonempty_tag('code', 'data.list.users'); - - $mform->addElement('text', 'config_arrayexpression', get_string('reader_json:arrayexpression', 'tool_dataflows')); - $mform->addElement('static', 'config_arrayexpression_help', '', - get_string('reader_json:arrayexpression_help', 'tool_dataflows', - ['jsonexample' => $jsonexample, 'expression' => $expression])); - - // JSON array sort by. - $mform->addElement('text', 'config_arraysortexpression', get_string('reader_json:arraysortexpression', 'tool_dataflows')); - $mform->addElement('static', 'config_arraysortexpression_help', '', - get_string('reader_json:arraysortexpression_help', 'tool_dataflows', - html_writer::nonempty_tag('code', 'userdetails.firstname'))); - - // JSON array sort order (ASC, DESC). - $mform->addElement('select', 'config_sortorder', get_string('reader_json:sortorder', 'tool_dataflows'), - ['asc' => get_string(self::ASC), 'desc' => get_string(self::DESC)]); - $mform->addElement('static', 'config_sortorder_help', '', get_string('reader_json:sortorder_help', 'tool_dataflows')); - } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index 9c6efb79..562d9755 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -137,10 +137,12 @@ $string['step_name_flow_abort'] = 'Abort'; $string['step_name_flow_append_file'] = 'Append'; $string['step_name_flow_copy_file'] = 'Copy File'; +$string['step_name_flow_curl'] = 'Curl'; $string['step_name_flow_email'] = 'Flow email notification'; $string['step_name_flow_file_put_content'] = 'File put content'; $string['step_name_flow_gpg'] = 'GPG'; $string['step_name_flow_hash_file'] = 'Hash file'; +$string['step_name_flow_json'] = 'JSON'; $string['step_name_flow_set_variable'] = 'Set variable'; $string['step_name_flow_logic_join'] = 'Join'; $string['step_name_flow_logic_switch'] = 'Switch'; diff --git a/lib.php b/lib.php index 8431bb4f..03676c81 100644 --- a/lib.php +++ b/lib.php @@ -69,10 +69,12 @@ function tool_dataflows_step_types() { new step\flow_abort, new step\flow_append_file, new step\flow_copy_file, + new step\flow_curl, new step\flow_email, new step\flow_file_put_content, new step\flow_gpg, new step\flow_hash_file, + new step\flow_json, new step\flow_logic_join, new step\flow_logic_switch, new step\flow_noop, diff --git a/version.php b/version.php index f560284b..c801ee84 100644 --- a/version.php +++ b/version.php @@ -25,7 +25,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023050401; +$plugin->version = 2023051800; $plugin->release = 2022102600; $plugin->requires = 2017051500; // Our lowest supported Moodle (3.3.0). $plugin->supported = [35, 401]; // Available as of Moodle 3.9.0 or later.