diff --git a/src/configs/Createdb.php b/src/configs/Createdb.php index 4d92c4973..3b2d2810f 100755 --- a/src/configs/Createdb.php +++ b/src/configs/Createdb.php @@ -954,6 +954,16 @@ $media_sources = [ 'https://pa.tedcdn.com/feeds/talks.rss', '############enclosure###Public@Podcast Examples/Ted/%Y-%m-%d %F', 'en-US'], + ['100000006', 'IMDB', 'description_source', 'video', + 'https://www.imdb.com/find?q=', + "Plot | span | data-testid = plot-l\n" . + "Genres | a | class = ipc-chip" . + "###li | class = find-result-item###" . + "a | class = ipc-metadata-list-summary-item__t###" . + "a | class = ipc-metadata-list-summary-item__t############" . + "Catch me if you can.mp4\n" . + "Inception.mp4\n" . + "Wrong Turn.mp4", 'en-US'] ]; $sql = "INSERT INTO MEDIA_SOURCE(TIMESTAMP, NAME, TYPE, CATEGORY, SOURCE_URL, AUX_INFO, LANGUAGE) VALUES (?, ?, ?, ?, ?, ?, ?)"; diff --git a/src/configs/PublicHelpPages.php b/src/configs/PublicHelpPages.php index 6b7305378..c96485163 100644 --- a/src/configs/PublicHelpPages.php +++ b/src/configs/PublicHelpPages.php @@ -2568,12 +2568,8 @@ page_header= page_footer= -page_theme= - page_type=standard -properties= - robots= share_expires=-2 @@ -3177,12 +3173,10 @@ resources are available for a page. ==Page Settings, Page Type== -: In edit mode for a wiki page, next to the page name, is a ⚙ link which can be used -to toggle settings controls. Clicking this link expands a form which can be used to -control global settings for a wiki page. This form contains a drop down for the -page type, another drop down for the type of border for the page in non-logged in mode, -a dropdown that let's you select from the available CSS themes for wiki pages on the -site (new themes can be created by Admin users with access to the Appearance Activity), +: In edit mode for a wiki page, next to the page name, is a link [Settings]. +Clicking this link expands a form which can be used to control global settings +for a wiki page. This form contains a drop down for the page type, another +drop down for the type of border for the page in non-logged in mode, a checkbox for whether a table of contents should be auto-generated from level 2 and level three headings and then text fields or areas for the page title, author, meta robots, and page description. @@ -3220,677 +3214,9 @@ on a line by itself is used to separate one slide. If presentation type is a sel slide icon appears in the wiki edit bar allowing one to easily add new slides. When the Activity panel is not collapsed and you are reading a presentation, it just displays as a single page with all slides visible. Collapsing the Activity panel presents -the slides as a typical slide presentation using the Javascript program: +the slides as a typical slide presentation using the javascript program: [[www.w3.org/Talks/Tools/Slidy2/Overview.html|Slidy]] -: The '''Share Wall''' page type allows you to make a page with a public textarea where people -can copy and paste text snippets for others to see. Above this text area is a formatted version -of the text with line numbers. - -: The '''URL Shortener''' page type allows you to do a page alias to an external -website rather than to a Yioop wiki page. The edit mode for this page lets you -see a graphs of how frequently people are following your link. - -: Page types of the form '''template:''' . '''some_names''' these are template pages. Templates consists of simple web form used to create a wiki page of a given type. For example, a genealogy template might have fields to fill in information about a new person one wished to add to a collection of genealogy wiki pages. -We describe how new templates (rather than pages built using an existing template) can be created in the next section. - -==Template Pages and Wiki Forms== - -: Template pages are wiki pages used to create other wiki pages. Any wiki page whose name begins with ''template:'' is a template page. All standard wiki page Syntax can be used on a template page. In addition, -the following two notations have meaning: - - {{text|name_of_text_field|placeholder_text_for_textfield}} - -and - - {{area|name_of_textarea|placeholder_text_for_textarea}} -which we describe below. To use a template wiki page, a user can create a new wiki page with the name they desire. The user then selects settings for the pages, and chooses the template from the list of page types and clicks save. This will show the template page parsed to look like a web page. The {{text| notations will show as textfields, and the {{area| notations will be replaced with textareas. The user can then fill in these fields with the desired values for their new page, and click save. If this wiki page is now viewed in read rather than edit mode, one will see the template page, but where rather than see text fields and text areas, they will see the filled in values instead. - -: Template Page forms are designed to facilitate the creation of new wiki pages, not collect information from people in a group or the web. To do the latter one can use Wiki form syntaxes. These can be used on any wiki page. Below is a list of syntaxes and the form fields they correspond to: - {{`t`extfield|name of textfield| label for text field}} - {{`t`extarea|name of text area| label for text area}} - {{`c`heckbox|name of checkbox with box to the right of label| label for check box}} - {{l`c`heckbox|name of checkbox with box to the left of label| label for check box}} - {{`r`adio|name of radio button with button to the right of label| label for radio button}} - {{l`r`adio|name of radio button with button to the left of label| label for radio button}} - =Yioop Wiki Syntax= - -: Wiki syntax is a lightweight way to markup a text document so that -it can be formatted and drawn nicely by Yioop. -This page briefly describes the wiki syntax supported by Yioop. - -==Headings== -: In wiki syntax headings of documents and sections are written as follows: - -<nowiki> -= Level 1 = -== Level 2 == -=== Level 3 === -==== Level 4 ==== -===== Level 5 ===== -====== Level 6 ====== -</nowiki> - -and would look like: - -= Level 1 = -== Level 2 == -=== Level 3 === -==== Level 4 ==== -===== Level 5 ===== -====== Level 6 ====== - -==Paragraphs== -: In Yioop two new lines indicates a new paragraph. You can control -the indent of a paragraph by putting colons followed by a space in front of it: - -<nowiki> -: some indent - -:: a little more - -::: even more - -:::: that's sorta crazy -</nowiki> - -which looks like: - -: some indent - -:: a little more - -::: even more - -:::: that's sorta crazy - -==Horizontal Rule== -: Sometimes it is convenient to separate paragraphs or sections with a horizontal -rule. This can be done by placing four hyphens on a line by themselves: -<nowiki> ----- -</nowiki> -This results in a line that looks like: ----- - -==Text Formatting Within Paragraphs== -: Within a paragraph it is often convenient to make some text bold, italics, -underlined, etc. Below is a quick summary of how to do this: -===Wiki Markup=== -{| -|<nowiki>''italic''</nowiki>|''italic'' -|- -|<nowiki>'''bold'''</nowiki>|'''bold''' -|- -|<nowiki>'''''bold and italic'''''</nowiki>|'''''bold and italic''''' -|} - -===HTML Tags=== -: Yioop also supports several html tags such as: -{| -|<nowiki><del>delete</del></nowiki>|<del>delete</del> -|- -|<nowiki><ins>insert</ins></nowiki>|<ins>insert</ins> -|- -|<nowiki><s>strike through</s> or -<strike>strike through</strike> </nowiki>|<s>strike through</s> -|- -|<nowiki><sup>superscript</sup> and -<sub>subscript</sub></nowiki>|<sup>superscript</sup> and -<sub>subscript</sub> -|- -|<nowiki><tt>typewriter</tt></nowiki>|<tt>typewriter</tt> -|- -|<nowiki><u>underline</u></nowiki>|<u>underline</u> -|} - -===Spacing within Paragraphs=== -: The HTML entity -<nowiki>&nbsp;</nowiki> -can be used to create a non-breaking space. The tag -<nowiki><br></nowiki> -can be used to produce a line break. - -==Preformatted Text and Unformatted Text== -: You can force text to be formatted as you typed it rather -than using the layout mechanism of the browser using the -<nowiki><pre>preformatted text tag.</pre></nowiki> -Alternatively, a sequence of lines all beginning with a -space character will also be treated as preformatted. - -: Wiki markup within pre tags is still parsed by Yioop. -If you would like to add text that is not parsed, enclosed -it in <tt><`mbox{nowiki}`> </`mbox{nowiki}`></tt> tags. - -==Styling Text Paragraphs== -: Yioop wiki syntax offers a number of templates for -control the styles, and alignment of text for -a paragraph or group of paragraphs:<br /> -`{{`left| some text`}}`,<br /> `{{`right| some text`}}`,<br /> -and<br /> -`{{`center| some text`}}`<br /> can be used to left-justify, -right-justify, and center a block of text. For example, -the last command, would produce: -{{center| -some text -}} -If you know cascading style sheets (CSS), you can set -a class or id selector for a block of text using:<br /> -`{{`class="my-class-selector" some text`}}`<br />and<br /> -`{{`id="my-id-selector" some text`}}`.<br /> -You can also apply inline styles to a block of text -using the syntax:<br /> -`{{`style="inline styles" some text`}}`.<br /> -For example, `{{`style="color:red" some text`}}` looks -like {{style="color:red" some text}}. - -==Lists== -: The Yioop Wiki Syntax supported of ways of listing items: -bulleted/unordered list, numbered/ordered lists, and -definition lists. Below are some examples: - -===Unordered Lists=== -<nowiki> -* Item1 -** SubItem 1 -** SubItem 2 -*** SubSubItem 1 -* Item 2 -* Item 3 -</nowiki> -would be drawn as: -* Item 1 -** SubItem 1 -** SubItem 2 -*** SubSubItem 1 -* Item 2 -* Item 3 - -===Ordered Lists=== -<nowiki> -# Item 1 -## SubItem 1 -## SubItem 2 -### SubSubItem 1 -# Item 2 -# Item 3 -</nowiki> -# Item1 -## SubItem 1 -## SubItem 2 -### SubSubItem 1 -# Item 2 -# Item 3 - -===Mixed Lists=== -<nowiki> -# Item1 -#* SubItem 1 -#* SubItem 2 -#*# SubSubItem 1 -# Item 2 -# Item 3 -</nowiki> -# Item1 -#* SubItem 1 -#* SubItem 2 -#*# SubSubItem 1 -# Item 2 -# Item 3 - -===Definition Lists=== -<nowiki> -;Term 1: Definition of Term 1 -;Term 2: Definition of Term 2 -</nowiki> -;Term 1: Definition of Term 1 -;Term 2: Definition of Term 2 - -==Tables== -: A table begins with {`|` and ends with `|`}. Cells are separated with | and -rows are separated with |- as can be seen in the following -example: -<nowiki> -{| -|a||b -|- -|c||d -|} -</nowiki> -{| -|a||b -|- -|c||d -|} -Headings for columns and rows can be made by using an exclamation point, !, -rather than a vertical bar |. For example, -<nowiki> -{| -!a!!b -|- -|c||d -|} -</nowiki> -{| -!a!!b -|- -|c||d -|} -Captions can be added using the + symbol: -<nowiki> -{| -|+ My Caption -!a!!b -|- -|c||d -|} -</nowiki> -{| -|+ My Caption -!a!!b -|- -|c||d -|} -Finally, you can put a CSS class or style attributes (or both) on the first line -of the table to further control how it looks: -<nowiki> -{| class="wikitable" -|+ My Caption -!a!!b -|- -|c||d -|} -</nowiki> -{| class="wikitable" -|+ My Caption -!a!!b -|- -|c||d -|} -Within a cell attributes like align, valign, styles, and class can be used. For -example, -<nowiki> -{| -| style="text-align:right;"| a| b -|- -| lalala | lalala -|} -</nowiki> -{| -| style="text-align:right;"| a| b -|- -| lalala | lalala -|} - -==Math== - -: Math can be included into a wiki document by either using the math tag: -<nowiki> -<math> -\sum_{i=1}^{n} i = frac{(n+1)(n)}{2} -</math> -</nowiki> - -<math> -\sum_{i=1}^{n} i = frac{(n+1)(n)}{2} -</math> - -or by enclosing the math in backticks: - -<pre> -`[[1, -2],[3,4]]` -</pre> - -`[[1, -2],[3,4]]`. - -Rendering of math is done using [[https://www.mathjax.org/|MathJax]], making use of the [[https://en.wikipedia.org/wiki/ASCIIMathML|ASCIImathml]] extensions. - -==Links and Relationships== -: A hypertext link to another document can be inserted into a wiki page using -the chain link icon in the GUI. Alternatively, there are several techniques -for inserting a link into a page depending on whether the link is to a page -within the same wiki group, is a link to a page on a different wiki -group, or is a link to a different website. In addition to normal -hypertext links, Yioop also supports relationship links. - -'''Intra-Group Wiki Links''' use the syntax: -<nowiki> -[[name_of_wiki_page]] -or -[[name_of_wiki_page|text for the link]] -or -[[name_of_wiki_page#heading_or_id_on_page|text for the link]] -</nowiki> -for example, to make a link to this Syntax page one could write, -<nowiki> -[[Syntax|Yioop Wiki Syntax Page]] -</nowiki> -which would look like, - -[[Syntax|Yioop Wiki Syntax Page]] - -'''Inter-Group Wiki Links''' use the syntax: -<nowiki> -[[name_of-group@name_of_wiki_page|text for the link]] -</nowiki> - -'''Different Website Links''' use the syntax: -<nowiki> -[[website_url|text for the link]] -</nowiki> - -: Relationships are a generalized form of link. They are used to express -a more complicated linking between two wiki pages and have the syntax: - -<nowiki> -[[relationship_type|wiki_page_name|text for the link]] -</nowiki> - -: In the navigation dropdown for a Yioop wiki page there are items for -what links to the current page and what relates to the current page -based on the links and relationships a page belongs to. - -==Recent Places Dropdowns== -: You can add a dropdown that can allow users to navigate to recently visited -wiki pages using the syntax: - -<sub>`[`{recent_places}]</sub> - -This looks like: - -[{recent_places}] - -==Adding Resources to a Page== - -: Yioop wiki syntax supports adding search bars, audio, images, and video to a -page. The magnifying class edit tool icon can be used to add a search bar via -the GUI. This can also be added by hand with the syntax: - -<nowiki> -{{search:default|size:small|placeholder:Search Placeholder Text}} -</nowiki> - -This syntax is split into three parts each separated by a vertical bar |. The -first part search:default means results from searches should come from the -default search index. You can replace default with the timestamp of a specific -index or mix if you do not want to use the default. The second group size:small -indicates the size of the search bar to be drawn. Choices of size are small, -medium, and large. Finally, placeholder:Search Placeholder Text indicates the -grayed out background text in the search input before typing is done should -read: Search Placeholder Text. Here is what the above code outputs: - -{{search:default|size:small|placeholder:Search Placeholder Text}} - -: Image, video and other media resources can be associated with a page by dragging -and dropping them in the edit textarea or by clicking on the link click to select -link in the gray box below the textarea. This would add wiki code such as - -<sub>((resource`:`myphoto.jpg|Resource Description))</sub> - -to the page. Only saving the page will save this code and upload the resource to -the server. In the above ''myphoto.jpg'' is the resource that will be inserted and -Resource Description is the alternative text to use in case the viewing browser -cannot display jpg files. To add a resource -from a different wiki page belonging to the same group to the current wiki -page one can use a syntax like: - -<sub>((resource`:`Documentation:ConfigureScreenForm1.png|The work directory form))</sub> - -Here Documentation would be the page and ConfigureScreenForm1.png the resource. -You can also insert resources from a data-string using ''resource-data'' rather than -''resource''. For example: - -<sub>((resource-data`:`image/jpeg;base64,/9j/4AAQSkZJRg...rest of image data...|Seekquarry Logo))</sub> - -could be used to inline an image like: - -((resource-data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAASABIAAD/4QBYRXhpZgAATU0AKgAAAAgAAgESAAMAAAABAAEAAIdpAAQAAAABAAAAJgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAAtqADAAQAAAABAAAAewAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgAewC2AwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAGhoaGhoaLRoaLUAtLS1AV0BAQEBXbVdXV1dXbYRtbW1tbW2EhISEhISEhJ6enp6enri4uLi4z8/Pz8/Pz8/Pz//bAEMBICIiNTE1WjExWtiTeJPY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2P/dAAQADP/aAAwDAQACEQMRAD8A0dS1L7GBHGAZGGeegFZCQavejzSzAHpubaPyFRX5H9rHz/ublznpt4rsRgjI6UAcqbDV4BvRycf3XJ/Q1Z0/VneQW1394nAbpz6GuirA1HS5ri58622jI5yccjvwKAN+imrnaN3XHNRz3ENunmTMFH8/pQBNRWC2vW4OFjYj3IH9at2uq2tywQEox6Bu/wBD0oA06KKY7pGheQhVHUmgB9FYcmu2qnEasw9eAP1qSDWrSVtr5j9zjH5igDYrn9cuJ4DF5Llc7s4Ppit/rzXNeIesP0b+lAG7Zsz2kTuckopJ/CrNVLD/AI8of+ua/wAqbd30FmB5uST0C8n60AXaKrW1yLqPzUVlU9N3Gas0AFFFFABRRRQAUUUUAf/Q0dS00XoEkZ2yKMc9DWGG1XT/AJfnCj23L/X+ldNFfWk8vkxPubGeAf51coA5aHX5h/ro1b3U4/xrZtNStrs7UO1/7rdfw9ammsrW4/1sak+uOfzrl9SsPsMiSQk7WPGeoI5//VQB2VcXdytqF/syQgJA9gOv511NrMbizSY9WXn61xdlb3E8xW2IDBc9ccU15id7aG/GtvEu2NQBWdf28OwzxAKR1A6EVI1jqSffdVz6sB/SlbTNUdSrEEH3H+FaOUWrWMlCad7mzpV01zagucsh2k+vofyrG1i5kmuhaIflUgf8CPf8K1NJs7izWRZgAGwRg56VzsiTzam6RffMjYz/AJ9KyNma0EVvbrhQCe5PU1BeW8MyF0AVxyCO/saZLa6jCu6aRUHTJYD+lSCw1YjIYEH3H+Fa88drGPJLe5b0K6aSJrdjnZgr9D2/CoPEPWH6N/SpdL025tLkyygbSpHBz3FReIesP/Av6VkbF37dHZabCTy5jXav4dfpVGxsZL6T7beZKnkA/wAX/wBb/P1zlsLua0N22SFA2g9So9PYVv6Pe/aIfIkPzxjj3X1oA2AABgUtFFABRRRQAUUUUAFFFFAH/9GpdCXTNR81BkZLL6EHqP8APtXS22oWtyoKOA3908Gpp7eG5Ty5l3D+X0rDl0BScwy4Howz/hQBvvNEi7ndQB3JrltVvkvHSC2+YKevqTxgVMNAkzzKo+i//XrWtNMtrQ7wN7/3m7fT0oAsWkJt7WOA9VXB+veuXRjpepncPkyf++W/z+ldjVS7soLxAsw5HQjqKAMfVbea9aOe1HmpjHGKmur1tPs47cMDPtA9cep/wqH+w5UJ8mfAPsR/I1Lb6HEj77h/M5zgDA/HuaAL2mzXNxb+dcY+Y/LgY49awtSSWz1EXSDhiHH1HUV1gAAwOAKingiuI/LmXI/l9KAMK/jbVEinsiH25BGQCCcetXPPXTNPRJSDIFwFz1P+Aqo2hMj7reYj6jn8wRSx6EC++4lLfQcn8Tk0AWNKury63PPjYvAOMZNUvEPWH6N/SuiiijhQRxAKo6AVSv8AT1vihZyuzPQZ60ATWP8Ax4w/9c1/lXN3sEmmXizwcITlfT3H+e30rqoIhDCkIOdihc/So7u1ju4TDJ9QfQ+tADredLmFZo+jDp6e1T1nWNgbHcFkLK3YjHNaNABRRRQAUUUUAFFFFAH/0unooooAKjkljhTfKwVR3NNuJ0t4Wmk6KM//AFq5HF1q9wWJwF/JR7UAbkmt2afc3v8AQf44pY9asn+9uT/eH+FRRaXZxj5l3n1akl0q0kHyDyz6r/hSuVym0jpIoeMhgehFPri45bnSbnY3KnkgdGH+NdhHIksayxnKsMimSSUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFAH//T6eiiigDntflKxxwj+Ilj+HT9TU2mRCKzQ93+Y/jVLxDnfH/un+YrUs/+PSL/AHB/Kky4lmimSSJEhkkOFHU1yt3qE9zMFhJVc4UA4P4+9IpuxsavCJLXzMcxnP4Hg1JoUpa1aI/wNx9Dz/Oi63jTG845fZz9areHw2Z27fL/AFpoiR0tFU7q+t7PHnE5PQDk1PDNHcRiWI5U0ySlqN+LKIbRl2+6D0+prIjj1i/QS+YUVjkZO3I9gOf1rbudPtrtg8wOQMcEjipXurSH5XkRcdsigDnJoNXsUM3mMyjqQxOPwNbGl37XsbCTAdDzjuD0NR3GrWHlOoYvxjgHHPv0rP8AD5zJJ/uj+ZoA0dZuJba3R4WKkvg49MGstr29vY0gtNxIQFyOCT9ewq74g/49Y/8AroP5Gp9EjVbEOvV2JP4HH9KAIdKj1FJWW7LbQOjc5J9D7VQGrTwXM3msWC7gq9s54rrK421iEur4YZHmMfyJP86AHFNaY/aB5nqOQP8Ax3P9KtahcX1lLH+9JDKDjjkjqOneumrK1iDzbNnHWP5h9O/6UAaUciyxrInRgCPxpJZFhiaV+igk/hWTok/mWphJ5jOB9DyKTXJ/LthAOsh5/wB0cmgCHS5ry8kd5ZGCKOnuef0FbXkt/wA9GqrpcBgs0DD5n+Y/j/8AWrRoA//U6eiiigDE1yAyWwmUZMZ5+hqHSbhZLfyCfmj4/DtW+yhlKsMg8EVyl3ptzZS+fZ5Kjpjkj6juKBp2LWrQXMyoIRlRnIHr2NYtxp09rGsp5HfHatKPWmAxMgJ9QcfoaZPq7SqY4owN3HJz+gpFOxUl1F5rMWz8tkZJ7gVqWk6abpwkfmSU7lX26D8K5+a1nt0V5V27ugPXH0oZ5ruTdIST3Jo2Ek5OyCWSa7kaRzknk1q6JeeTN9nf7sn6N2/PpVZECLtWqcgMc25eM8j6/wD66iM7ux01cPyQUjf1e8maf7FAT2BxwST0FTwaFCqgzuS3cLwP8azr+OUtHqcY+WRVYn0YetaUevW2z96rBgOcYIrQ5C02nWFvE8nlg4UnLc9PrWXoH+tk/wBxf5mnzXV1qv8Ao9ohSM/eY9/xH9Kh0NvKu3hbqVI/FTQBe1//AI9o/wDroP5GrWkf8g6L8f5mquv/APHtH/10H8jVrSP+QdF+P8zQBp1yen/8hdv96T+ddZXJ6fj+12/3pP50AdZSEAgg9DS0UAcnYk2OqNbseCSn9Vpbn/T9XEI5VTtP0Xlv8Km1yJo5I7yPg9Cfccin6HEzGS7k5JO0fzP60AdDS0UUAf/V6eiiigAooooAhkt7eX/Wxq31ANUrqa002LcqKGP3VUAZP+FWbu6jtITNJ9APU+lcNNNNezl5Dlj+QHoPagaTbsglkmvZjJIck9+wH+FWkRUXatCIqLgU6ueUrnrUKCgrvcKp3DAyADtViWQRr79qfplk15cbnHyLyx9fb8aqnHqZYuqrciOssEKWUSOOQoyKkNpal95iTd67Rmp+nFLWx5ogAAwOKTauc4GadRQAhAPUZoAA4FLRQAU0KoOQBTqKACkyKWsvU7GS8VDEwVkz1z3+lAGfrV0HKWcJ3HOWA9ew/Otu0gFtbJCP4Rz9e9Zdho4tpBNOwZh0A6A+vvW7QAUUUUAf/9bp6KKKACiiigCnfWaXsPlMcEHII9a5W40a7g+ZBvHqOv5V21FAHnQlmjO1uo7HrTzctjhcGu4uLa3lX94it9RWcmn2W/8A1S1PKjZV5pWTOes7Ce+kz0Xuxrtbe3jtohFEMAVKqqg2qMD2p1UYthRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQB//9k=|The Seekquarry Logo)) - -be aware though that the default maximum wiki page size is 512Kb (this can be set in src/configs/Config.php). - -: Sometimes it is useful to edit the basic resource link -above to make a link which is a thumbnail of the resource which points to a -separate page containing that resource. This can be done using the syntax: - -<sub>((resource-thumb`:`myphoto.jpg|Resource Description))</sub> - -: Similarly, by default for resources like PDFs, epub's, etc., the resource tag inlines -the whole resource into the page, if instead one wants a clickable link to a page where -the resource is displayed one can use the syntax: - -<sub>((resource-link`:`my_document.pdf|Resource Description))</sub> - -: Comma separated value files (.csv or CSV files) are inlined into a page as a table. Which rows and columns of the CSV to present in this table can be controlled by the resource line. The general format for including -a CSV resource is: - -<sub> ((resource`:`resource_name.csv#config#top_left_cell#bottom_right_cell|Resource Description))</sub> - -For example, - -<sub>((resource`:`resource_name.csv##B2#C3|Resource Description))</sub> - -might output - -((resource-data:text/csv;base64,LCwsLAosLTIsMywsCiw1LDQsLAosLCwsCiwsLCwK##B2#C3|Example CSV with Headings)) - -I.e., just the portion of the CSV given by the rectangle between the cells B2 and C3. Using a config directive we can omit the spreadsheet row and column headings as follows: - -<sub>((resource`:`resource_name.csv#noheadings#B2#C3|Resource Description)) </sub> - -which might output - - -((resource-data:text/csv;base64,LCwsLAosLTIsMywsCiw1LDQsLAosLCwsCiwsLCwK#noheadings#B2#C3|Example CSV without Headings)) - -CSV spreadsheet files can also be used to output a variety of charts. The general format for the command to insert a chart resource is: - -<sub>((resource-chart_type`:`resource_name.csv#char_config#x_start#x_end#y_start#y_end|Resource Description))</sub> - -Here ''chart_type'' can be one of ''bargraph'', ''linegraph'', or ''pointgraph''. For example, one might have a line like: - -<sub>((resource-bargraph`:`resource_name.csv##B1#B4#C1#C4|Quadratic Function)) </sub> - -which could produce a chart like - -((resource-bargraph:##(1,1)#(2,4)#(3,9)#(4,16)|Quadratic Function)) - -In the above example, the values for the `x` coordinates would come from the cells B1, B2, B3, B4 from -''resource_name.csv '' and the values for the `y` coordinates would come from cells C1, C2, C3, C4 from -''resource_name.csv ''. Alternatively, rather than use a CSV to get out data we can just list the points we want to plot with a command like: - -<sub>((resource-bargraph`:`##(1,1)#(2,4)#(3,9)#(4,16)|Quadratic Function))</sub> - -: If the QRENCODE variable has been set in src/configs/LocalConfig.php to the path of a working -copy of the qrencode command, then QR code resources can be displayed on wiki pages. Such a resource -has the format: - -<sub>((resource-qr`:`url-to-qr-encode|A description of the qr code))</sub> - -For example, if Yioop has been appropriately configured below one should see a QR code for https://www.seekquarry.com/ - -((resource-qr:https://www.seekquarry.com/|Seekquarry)) - -==Manipulating Page Resources== - -: A list of media that have already been associated with -a page appears under the Page Resource heading below the textarea. This -table allows the user to rename and delete resources as well as insert the -same resource at multiple locations within the same document. - -: The resources section of the edit page can be thought of as similar to -a folder in Windows or MacOS. One can have subfolders of the resource folder. - -: The '''Places''' dropdown at the top of the '''Page Resource''' section name has the -name of the current subfolder of the resource folder one is in, and allows you to navigate -up folders to the root resource folder. - -: The '''View''' selection gadget allows you to control whether you want to view the resource -items in the current folder as a list, as a grid, or in a detail mode. If you are in detail mode -then the currently selected appears in a "details" top panel; whereas, all other items in the folder -appear beneath it in a horizontal list. the details item consists of an icon, size, modified date, etc details. One can also configure to have a short text description display under the item. To do this enter edit -mode, click on an item and type the description. - -: The '''Sort''' dropdown (up-down arrows) allows you to control the order in which media lists items are presented. - -: The '''Filter''' textfield lets you enter a search string. -Clicking '''Go''' then shows only those resources -which contain that search string in their title. - -: The '''Clipboard ''' icon is used to toggle showing the user's clipboard. The clipboard can be -used to move items around the folder structure. In edit mode, when a resource item is clicked on, -it becomes a text field with three options: Rename, Copy Clipboard, and Cut Clipboard. The latter -two can be used to either copy the item to the clipboard folder or to move it there. Once in the -clipboard, the user can move to a different folder and paste the item as desired. Alternatively, -the user can delete the clipboard's contents. - -: The '''Actions''' dropdown (labelled ...) can be used to create new folders, new text files, and new csv -text files within the current page resource folder. These are initially named beginning -with ''untitled'' followed by some number, and if applicable a file extension. In the dropdown there is also an option to upload a file to the current resource folder. - -: The '''Name''', '''Size''', '''Modified''' header links above the resources list -control the sort order for the resource list. If a page is a media list page, -then even in read mode, the sort order selected is remembered when drawing the -media list. - -: Resources entries for the resources list consist first of an icon. In edit mode, when clicked on, -they become a textfield with a name for the resource, followed by buttons for actions -that can be done to that resource (Rename, Add to Page, Clip Cut, Clip Copy). If a resource is -editable the icon will look like a plus sign together with a pencil. Clicking -on the icon will then let you edit the resource. - -===Text and CSV Resources=== -: For normal text files clicking edit will bring up a textarea with the context of the text to edit. -For CSV (comma separated value) files this will present the file as an editable spreadsheet. -Yioop spreadsheets can have equation much like Excel spreadsheets. Clicking on a cell lets one -edit its contents. For example, if in the cell A3 -one entered the equation: - = A1+A2 -then clicking out of the cell would cause it to refresh with the value of the sum of the contents of -cells A1 and A2. In addition, to the standard arithmetic operators ['*', '/', '+', '-', '%'], the -spreadsheet expressions can use float or integer literals, and can make use of the following table -of built-in functions: - -{| class="wikitable" -!Function Name!!Description -|- -!avg(x1,...,xn), avg(x1:xn)|| computes average of values of cells listed as arguments -|- -!ceil(x)|| rounds the value of x up to nearest integer -|- -!cell(i,j)|| returns the contents of the cell with column name of letter j, and row name i. For example, cell(2,'B') would return the contents of cell B2. -|- -!col(value, search_row, start_col, end_col)|| searches the row ''search_row'' between the columns -''start_col'', ''end_col'' for ''value''. Returns the column name where this value was found or -1 if not found. -For example, col(3, 2, "B", "D") might return C if the cell C2 had value 3. -|- -!exp(x)|| computes `e^x` -|- -!floor(x)|| rounds the value of x down to the nearest integer -|- -!log(x)|| computes `log x` -|- -!min(x1,...,xn), min(x1:xn)|| computes minimum value of cells listed as arguments -|- -!max(x1,...,xn), max(x1:xn)|| computes maximum value of cells listed as arguments -|- -!pow(x,y)|| computes `x^y` -|- -!row(value, search_col, start_row, end_row)|| searches the column ''search_col'' -between the rows ''start_row'', ''end_row'' for ''value''. -Returns the row name where this value was found or -1 if not found. -For example, row(3, "C", "1", "5") might return 2 if the cell C2 had value 3. -|- -!sqrt(x)|| computes `sqrt(x)` -|- -!sum(x1,...,xn), sum(x1:xn)|| computes sum of values of cells listed as arguments -|- -!username()|| returns username of the person using this CSV file -|} - -===HTML, PDF and EPub Resources=== -: How HTML, PDF, EPub resources included on a page render depends on how the Yioop wiki software -has been configured. If no special configuration has been done, then HTML and PDF documents -will bbe rendered in an <iframe> tag within the current wiki page. In the EPub, case a link -to download the resource will be given. If the wiki software detects the presence of the -file APP_DIR/scripts/epubjs-reader ([[https://github.com/futurepress/epubjs-reader/|epubjs-reader]]), the wiki -system will render the resource in a Javascript viewer and will do things like remember reading -position. - -===Video and Audio Resources=== - -: Not all browsers support the same video and audio formats for playback. For this reason -it sometimes is useful to have multiple video resources for the same video. For example, -you might have a .ogv and .vp8 version of the same video recording. In read (non-edit) -mode, the Yioop wiki system displays only one link for video or audio files that have -the same name except for extension. It then includes the grouped file as separated <source> -tags within either the <video> or <audio> html tag used to render the item in the browser. -In this way, you can make your media take best advantages to whatever capabilities your -client's browser has. If you don't feel like recoding your media in such a fancy way, a safe -rule of thumb is that .mp3 audio will playback in all modern browser, and that .mp4 video -will playback in all modern browser. - -: For video it is sometimes useful to add a subtitle or caption track. Yioop wiki supports -[[https://en.wikipedia.org/wiki/WebVTT|WebVTT]] format subtitles and captions. To see how -Yioop wiki makes use of these files, suppose you included a resource ''foo.mp4'' in your -wiki pages, and you also had a file named ''foo-captions-en-US.vtt'' then when the HTML -page is generated from your wiki page, a <track> tag for the caption file would be added -to the <video> tag. A user seeing this page would then see in the video player a closed caption -symbol and be able to turn on/off (defaults off) the English captions. If you wanted -named the file ''foo-subtitles-en-US.vtt'' instead, then Yioop wiki would include it as a -subtitles track (defaults on). You can add captions/subtitle files for as many languages as -desired. - -: When viewing the page resources for a page in edit mode, one can see one file/resource and -no grouping of resources by name is done. In this way you can keep track of exactly what -resources are available for a page. - -==Page Settings, Page Type== - -: In edit mode for a wiki page, next to the page name, is a ⚙ link which can be used -to toggle settings controls. Clicking this link expands a form which can be used to -control global settings for a wiki page. This form contains a drop down for the -page type, another drop down for the type of border for the page in non-logged in mode, -a dropdown that let's you select from the available CSS themes for wiki pages on the -site (new themes can be created by Admin users with access to the Appearance Activity), -a checkbox for whether a table of contents should be auto-generated from level 2 -and level three headings and then text -fields or areas for the page title, author, meta robots, and page description. -Beneath this one can specify another wiki page to be used as a header for this -page and also specify another wiki page to be used as a footer for this page. - -: The contents of the page title is displayed in the browser title when the -wiki page is accessed with the Activity Panel collapsed or when not logged in. -Similarly, in the collapsed or not logged in mode, if one looks as the HTML -page source for the page, in the head of document, <meta> tags for author, -robots, and description are set according to these fields. These fields can -be useful for search engine optimization. The robots meta tag can be -used to control how search engine robots index the page. Wikipedia has more information on -[[https://en.wikipedia.org/wiki/Meta_element|Meta Elements]]. - -: The '''Standard''' page type treats the page as a usual wiki page. - -: '''Page Alias''' type redirects the current page to another page name. This can -be used to handle things like different names for the same topic or to do localization -of pages. For example, if you switch the locale from English to French and -you were on the wiki page dental_floss when you switch to French the article -dental_floss might redirect to the page dentrifice. - -: '''Media List''' type means that the page, when read, should display just the -resources in the page as a list of thumbnails and links. These links for the -resources go to a separate pages used to display these resources. -This kind of page is useful for a gallery of -images or a collection of audio or video files. For videos, thumbnails will appear -as animated gifs, provided the FFMPEG constant has been defined in the system's -src/configs/LocalConfig.php and it is set to the path of an executable of ffmpeg. - -: '''Presentation''' type is for a wiki page whose purpose is a slide presentation. In this mode, -.... -on a line by itself is used to separate one slide. If presentation type is a selected a new -slide icon appears in the wiki edit bar allowing one to easily add new slides. -When the Activity panel is not collapsed and you are reading a presentation, it just -displays as a single page with all slides visible. Collapsing the Activity panel presents -the slides as a typical slide presentation using the Javascript program: -[[www.w3.org/Talks/Tools/Slidy2/Overview.html|Slidy]] : The '''Share Wall''' page type allows you to make a page with a public textarea where people can copy and paste text snippets for others to see. Above this text area is a formatted version @@ -3900,36 +3226,6 @@ of the text with line numbers. website rather than to a Yioop wiki page. The edit mode for this page lets you see a graphs of how frequently people are following your link. -: Page types of the form '''template:''' . '''some_names''' these are template pages. Templates consists of simple web form used to create a wiki page of a given type. For example, a genealogy template might have fields to fill in information about a new person one wished to add to a collection of genealogy wiki pages. -We describe how new templates (rather than pages built using an existing template) can be created in the next section. - -==Template Pages and Wiki Forms== - -: Template pages are wiki pages used to create other wiki pages. Any wiki page whose name begins with ''template:'' is a template page. All standard wiki page Syntax can be used on a template page. In addition, -the following two notations have meaning: - - {{text|name_of_text_field|placeholder_text_for_textfield}} - -and - - {{area|name_of_textarea|placeholder_text_for_textarea}} -which we describe below. To use a template wiki page, a user can create a new wiki page with the name they desire. The user then selects settings for the pages, and chooses the template from the list of page types and clicks save. This will show the template page parsed to look like a web page. The {{text| notations will show as textfields, and the {{area| notations will be replaced with textareas. The user can then fill in these fields with the desired values for their new page, and click save. If this wiki page is now viewed in read rather than edit mode, one will see the template page, but where rather than see text fields and text areas, they will see the filled in values instead. - -: Template Page forms are designed to facilitate the creation of new wiki pages, not collect information from people in a group or the web. To do the latter one can use Wiki form syntaxes. These can be used on any wiki page. Below is a list of syntaxes and the form fields they correspond to:<br> -`{{`textfield|name of textfield| label for text field`}}`<br> -`{{`textarea|name of text area| label for text area`}}`<br> -`{{`checkbox|name of checkbox with box to the right of label| label for check box`}}`<br> -`{{`lcheckbox|name of checkbox with box to the left of label| label for check box`}}`<br> -`{{`radio|name of radio button with button to the right of label| label for radio button`}}`<br> -`{{`lradio|name of radio button with button to the left of label| label for radio button`}}`<br> -`{{`submit|name of submit button| value/text appearing on button`}}`<br> -`{{`image-captcha|placeholder for captcha text field`}}`<br> -`{{`username|name of hidden form field to store user's username in`}}`<br> -`{{`date|name of form field to store date form sent to end user to fill out`}}`<br> -`{{`timestamp|name of form field to store Unix timestamp of when form sent to end user to fill out`}}`<br> -If a form is to be processed by Yioop, at a minimum in needs to have an image-captcha and a submit button. -For textfield's, textarea's, checkboxes, and radion button's each of the syntax also has a required variant beginning with either r- or required-. For example, `{{`r-textfield|name of textfield| label for text field`}}`. A user filling out a wiki field must provide a value for a required item for that form to be processed. When a user fills in and submits a Wiki page with a form on it, Yioop checks if the captcha was filled in correctly, if not the data is rejected. If it was filled in correctly, and values were given for all required fields, then a new row with the form data is added to the form_data.csv file in that Wiki page's list of page resources. - EOD; $public_pages["en-US"]["ad_program_terms"] = <<< 'EOD' page_type=standard @@ -4087,17 +3383,11 @@ url_shortener= END_HEAD_VARS== Suggest URL Timeout == The number of allowed suggestion per day has been exceeded. Please wait till %s to make a new url suggestion. -EOD; -$public_pages["en-US"]["template:test"] = <<< 'EOD' - EOD; $public_pages["en-US"]["terms"] = <<< 'EOD' =Terms of Service= Please write the terms for the services provided by this website. -EOD; -$public_pages["en-US"]["test"] = <<< 'EOD' - EOD; $public_pages["es"]["400"] = <<< 'EOD' page_type=page_alias @@ -42763,6 +42053,43 @@ For each machine listed under Machines, there is a delete link to remove it from {{right|[[https://www.seekquarry.com/?c=static&p=Documentation#GUI%20for%20Managing%20Machines%20and%20Servers| Learn More.]]}} EOD; +$help_pages["en-US"]["Main"] = <<< 'EOD' +alternative_path= + +author= + +default_sort=aname + +description= + +page_alias= + +page_border=solid-border + +page_header= + +page_footer= + +page_theme= + +page_type=standard + +properties= + +robots= + +share_expires=-2 + +title= + +toc=true + +url_shortener= + +update_desc=true + +END_HEAD_VARSCreating Main Page +EOD; $help_pages["en-US"]["Manage_Advertisements"] = <<< 'EOD' page_type=standard @@ -42917,29 +42244,39 @@ sort=aname END_HEAD_VARSThe '''Max Depth''' dropdown is used to limit what urls are allowed to be crawl by the number of hops they are from a seed site. For example, if the Max Depth was set to 2, then seed sites would be crawled, sites linked to seed sites would be crawled, and sites linked to sites linked to seed sites would be crawled. A depth 0 crawl only crawls the seed sites. EOD; $help_pages["en-US"]["Media_Sources"] = <<< 'EOD' -page_type=standard +alternative_path= + +author= + +default_sort=aname + +description= page_alias= page_border=solid-border -toc=true +page_header= -title= +page_footer= -author= +page_theme= + +page_type=standard + +properties= robots= -description= +share_expires=-2 -alternative_path= +title= -page_header= +toc=true -page_footer= +url_shortener= -sort=aname +update_desc= END_HEAD_VARS'''Media Sources''' are used to specify how Yioop should handle news feeds, podcast, and trending value sites. The '''Add Media Source''' form lets you add new media sources. What this form looks like depends on the '''Type''' dropdown chosen. Below we describe the form for each of the possible choices of type: @@ -43062,6 +42399,8 @@ This url is then downloaded and a string matching the pattern /window\.\_\_data ]+\}\;) is then converted to a JSON object, because of the json| in the Aux Url XPath. From this JSON object, we look at the video field, then the current subfields, its 0 subfield, and finally, the publicUrl field. This is the url we download next. Lastly, the download XPath is then used to actually get the final video link from this downloaded page. Once this video is downloaded, it is stored in the Podcasts page's resource folder of the the My Private Group wiki group in a file with a name in the format: %Y-%m-%d.mp4. +<br/> + A '''Trending value source''' is a value on a web page that one would like to track using Yioop's trending search mechanism. The Name field is the name to use for the trending value. The URL field should be the page with the periodically updated value. '''Category''' should be the trends category (a collection of trending values) one would like to track this value with. '''Group Within Category''' is the default name of the key that will be associated with the value found on this page. '''Trend Value Regex''' is a regular expression to match against the downloaded URL. If it matches and the expression has one capture group, then tat capture group will be used as the value for a particular download time. If it has two or more capture groups, the first two capture groups are used to give a key name, value pair for a particular download time. As an example, Name: Yioop Ticker @@ -43073,6 +42412,24 @@ A '''Trending value source''' is a value on a web Here there is only one capture group (\d+\.\d+), so searching on trending:stocks, one would see all the hour, weekly, etc values for the trending values with that category. One such row would be Yioop Price whose values would be computed based on the numbers extracted according to this regex's (\d+\.\d+) capture group. +<br/> + +A '''Description Source''' is used to update the description of wiki page resources by looking using the resources name. The '''Name''' field is used to given logical name to the source. The '''URL''' field is used to provide the url of web page along with any required query parameters in order to look up resource using its name. The '''Language''' field is used to specify the locale to be used at search site given they support it. The '''Mime Type''' field is used to specify mime type of file format of resources to be processed. The '''Required Info Tags''' field is used to specify the details of HTML tags containing the required information to be collected as the description of the resources. The '''List Row Tag''' field is used to specify tag name and optionally attribute with value that aids to uniquely identify the HTML elements that completely contain all the details of a single search result, mostly this will be a <tr> tag. The '''Row Title Tag''' field is used to specify the details of HTML tag within the '''List Row Tag''' that contains the text representing the title of search result in the similar format as '''List Row Tag'''. The '''Row URL Tag''' field is used to specify details of HTML tag within '''List Row Tag''' that contains the URL of details page about the search result. The '''Test Values''' field is used to provide test values to be used while in the test mode of search source. Below is the example of search source for IMDB site +<pre> + Name: IMDB + URL: https://www.imdb.com/find?q= + Language: English + Mime Type: VIDEO + Required Info Tags: + Storyline | div | data-testid = storyline-plot-summary + Details | div | data-testid = title-pc-wide-screen + List Row Tag: tr | class = findResult + Row Title Tag: td | class = result_text + Row URL Tag: a + Test Values: + Brahmastra Part One.mp4 + House of the Dragon.mp4 +</pre> EOD; $help_pages["en-US"]["Media_Updater"] = <<< 'EOD' page_type=standard diff --git a/src/controllers/components/CrawlComponent.php b/src/controllers/components/CrawlComponent.php index f353450b8..5e666c2d3 100644 --- a/src/controllers/components/CrawlComponent.php +++ b/src/controllers/components/CrawlComponent.php @@ -224,7 +224,7 @@ class CrawlComponent extends Component implements CrawlConstants $parent->web_site->filePutContents($filename, serialize($crawl_params)); chmod($filename, 0777); - if($crawl_model->sendStartCrawlMessage($crawl_params, + if ($crawl_model->sendStartCrawlMessage($crawl_params, null, $machine_urls)) { return $parent->redirectWithMessage( tl('crawl_component_resume_crawl'), @@ -686,7 +686,7 @@ class CrawlComponent extends Component implements CrawlConstants "/schedules/{$crawl_params[self::CHANNEL]}-NameServerMessages.txt"; $parent->web_site->filePutContents($filename, serialize($crawl_params)); chmod($filename, 0777); - if(!$crawl_model->sendStartCrawlMessage($crawl_params, + if (!$crawl_model->sendStartCrawlMessage($crawl_params, $seed_info, $machine_urls)) { $parent->redirectWithMessage( tl('crawl_component_start_fail'), @@ -2543,6 +2543,7 @@ class CrawlComponent extends Component implements CrawlConstants "feed_podcast" => tl('crawl_component_feed_podcast'), "scrape_podcast" => tl('crawl_component_scrape_podcast'), "trending_value" => tl('crawl_component_trending_value'), + "description_source" => tl('crawl_component_description_source') ]; $data['PODCAST_EXPIRES'] = [ -1 => tl('crawl_component_never'), @@ -2551,6 +2552,12 @@ class CrawlComponent extends Component implements CrawlConstants C\ONE_MONTH => tl('crawl_component_one_month'), C\ONE_YEAR => tl('crawl_component_one_year') ]; + $data['MEDIA_MIME_TYPE'] = [ + "video" => "VIDEO", + "audio" => "AUDIO", + "text" => "TEXT", + "application" => "APPLICATION" + ]; $source_type_flag = false; if (isset($_REQUEST['type']) && in_array($_REQUEST['type'], @@ -2641,7 +2648,7 @@ class CrawlComponent extends Component implements CrawlConstants "aux_info" => "", 'category' => "news", 'channel_path' => "", "image_xpath" => "", "trending_stop_regex"=> "", 'item_path' => "", 'title_path' => "", 'description_path' => "", 'link_path' => "", - "language" => $data['SOURCE_LOCALE_TAG']]; + 'language' => $data['SOURCE_LOCALE_TAG'], 'test_data' => ""]; $data["CURRENT_SUBSEARCH"] = [ "default_query" => "", "folder_name" =>"", "index_identifier" => "", "landing_highlight" => "highlight:false", @@ -2673,7 +2680,7 @@ class CrawlComponent extends Component implements CrawlConstants $must_have = ["name", "type", 'source_url']; if (isset($_REQUEST['type']) && !in_array($_REQUEST['type'], ['feed_podcast', - 'scrape_podcast'])) { + 'scrape_podcast', 'description_source'])) { $_REQUEST['trending_stop_regex'] ??= ""; } $is_parse_feed = false; @@ -2695,6 +2702,15 @@ class CrawlComponent extends Component implements CrawlConstants $_REQUEST['type'] == 'trending_value') { $must_have[] = 'image_xpath'; $is_parse_feed = true; + } else if (isset($_REQUEST['type']) && + $_REQUEST['type'] == 'description_source') { + $must_have = array_merge($must_have, [ + 'item_path', 'channel_path', 'test_data', + 'title_path', 'description_path']); + if (isset($_REQUEST['format'])) { + $_REQUEST['category'] = $_REQUEST['format']; + } + $is_parse_feed = true; } if (isset($_REQUEST['type']) && $_REQUEST['type'] == -1) { return $parent->redirectWithMessage( @@ -2705,7 +2721,7 @@ class CrawlComponent extends Component implements CrawlConstants ['aux_info', 'category','language', 'image_xpath', 'channel_path', 'item_path', 'title_path', 'trending_stop_regex', 'description_path', - 'link_path']); + 'link_path', 'test_data']); foreach ($to_clean as $clean_me) { $r[$clean_me] = (isset($_REQUEST[$clean_me])) ? trim($parent->clean($_REQUEST[$clean_me], @@ -2739,7 +2755,7 @@ class CrawlComponent extends Component implements CrawlConstants $r['item_path'] . "###" . $r['title_path']. "###".$r['description_path']."###".$r['link_path']. "###" . $r['image_xpath'] . "###" . - $r['trending_stop_regex']; + $r['trending_stop_regex'] . "###" . $r['test_data']; } else if (isset($_REQUEST['type']) && $_REQUEST['type'] == 'rss') { $r['aux_info'] = $r['image_xpath'] . "###" . @@ -2933,18 +2949,25 @@ class CrawlComponent extends Component implements CrawlConstants $_REQUEST['category'] = $_REQUEST['expires']; } } + if (isset($_REQUEST['type']) && + $_REQUEST['type'] == 'description_source') { + if (isset($_REQUEST['format'])) { + $_REQUEST['category'] = $_REQUEST['format']; + } + } $aux_parts = explode("###", $source['AUX_INFO']); if (in_array($source['TYPE'], ['html', 'json', 'regex', 'feed_podcast', 'scrape_podcast', - 'trending_value'])) { + 'trending_value', 'description_source'])) { $is_parse_feed = true; - $aux_parts = array_pad($aux_parts, 7, ""); + $aux_parts = array_pad($aux_parts, 8, ""); list($source['CHANNEL_PATH'], $source['ITEM_PATH'], $source['TITLE_PATH'], $source['DESCRIPTION_PATH'], $source['LINK_PATH']) = $aux_parts; $source['IMAGE_XPATH'] = $aux_parts[5] ?? ""; $source['TRENDING_STOP_REGEX'] = $aux_parts[6] ?? ""; + $source['TEST_DATA'] = $aux_parts[7] ?? ""; } else if ($source['TYPE'] == 'rss') { $is_rss_feed = true; $source['IMAGE_XPATH'] = $aux_parts[0] ?? ""; @@ -2973,7 +2996,8 @@ class CrawlComponent extends Component implements CrawlConstants $source['DESCRIPTION_PATH'] . "###". $source['LINK_PATH']. "###". $source['IMAGE_XPATH'] . "###" . - $source['TRENDING_STOP_REGEX']; + $source['TRENDING_STOP_REGEX'] . "###" . + $source['TEST_DATA']; } else if ($is_rss_feed) { $source['AUX_INFO'] = $source['IMAGE_XPATH'] . "###" . $source['TRENDING_STOP_REGEX']; @@ -2981,8 +3005,8 @@ class CrawlComponent extends Component implements CrawlConstants unset($source['CHANNEL_PATH'], $source['ITEM_PATH'], $source['TITLE_PATH'], $source['DESCRIPTION_PATH'], $source['LINK_PATH'], $source['IMAGE_XPATH'], - $source['TRENDING_STOP_REGEX']); - + $source['TRENDING_STOP_REGEX'], + $source['TEST_DATA']); $source_model->updateMediaSource($source); $fields = array_merge(array("arg", "ts"), $request_fields); @@ -3058,6 +3082,13 @@ class CrawlComponent extends Component implements CrawlConstants $data['FEED_TEST_RESULTS'] .= $wiki_update_job->updatePodcastsOneGo([$source], C\ONE_WEEK, true); + } else if ($source['TYPE'] == 'description_source') { + $desc_update_job = new M\DescriptionUpdateJob(); + $desc_update_job-> + parseDescriptionSourceAuxInfo($source); + $data['FEED_TEST_RESULTS'] = $desc_update_job-> + updateResourcesDescription([$source], + test_mode: true); } break; } @@ -3135,4 +3166,4 @@ class CrawlComponent extends Component implements CrawlConstants } return $data; } -} +} \ No newline at end of file diff --git a/src/controllers/components/SocialComponent.php b/src/controllers/components/SocialComponent.php index 05b98599f..d976510f9 100644 --- a/src/controllers/components/SocialComponent.php +++ b/src/controllers/components/SocialComponent.php @@ -62,6 +62,11 @@ class SocialComponent extends Component implements CrawlConstants * successfully uploaded */ const UPLOAD_SUCCESS = 1; + /** + * File to tell DescriptionUpdateJob that a wiki resource needs a + * description + */ + const NEEDS_DESC_FILE = C\APP_DIR . "/resources/needs_descriptions.txt"; /** * Used to handle the manage group activity. * @@ -3658,7 +3663,7 @@ class SocialComponent extends Component implements CrawlConstants fputcsv($fh, $csv_headers); } $out_row = []; - foreach($csv_headers as $csv_header) { + foreach ($csv_headers as $csv_header) { $is_required = (isset($_POST['RCSVFORM'][$csv_header])) ? 1 : 0; if ($is_required && empty($_POST[$csv_header])) { return $parent->redirectWithMessage( @@ -3904,6 +3909,12 @@ EOD; $data['RESOURCES_INFO'] = $group_model->getGroupPageResourceUrls($group_id, $data['PAGE_ID'], $sub_path); + $thumb_folder = $data['RESOURCES_INFO']['thumb_folder']; + if (!empty($thumb_folder) && !empty($data["HEAD"]['update_desc'])) { + $fp = fopen(self::NEEDS_DESC_FILE, "a"); + fwrite($fp, $thumb_folder . "\n"); + fclose($fp); + } $this->initUserResourcePreferences($data); $scroll_id = "scroll-container-" . L\crawlHash($data['PAGE_ID'] . $sub_path); @@ -4354,6 +4365,8 @@ EOD; } else { $head_vars[$key] == true; } + } else if ($key == 'update_desc') { + $head_vars[$key] = isset($_REQUEST['update_desc']); } } $head_string = ""; @@ -5057,6 +5070,7 @@ EOD; 'title' => '', 'toc' => true, 'url_shortener' => '', + 'update_desc' => false ]; /* Check if back params need to be set. Set them if required. the back params are usually sent when the wiki action is initiated @@ -5205,4 +5219,4 @@ EOD; } EOD; } -} +} \ No newline at end of file diff --git a/src/data/public_default.db b/src/data/public_default.db index 75ecc3063..d7df7adbf 100644 Binary files a/src/data/public_default.db and b/src/data/public_default.db differ diff --git a/src/library/media_jobs/DescriptionUpdateJob.php b/src/library/media_jobs/DescriptionUpdateJob.php new file mode 100644 index 000000000..415dc00f5 --- /dev/null +++ b/src/library/media_jobs/DescriptionUpdateJob.php @@ -0,0 +1,521 @@ +<?php +/** + * SeekQuarry/Yioop -- + * Open Source Pure PHP Search Engine, Crawler, and Indexer + * + * Copyright (C) 2009 - 2022 Chris Pollett chris@pollett.org + * + * LICENSE: + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + * END LICENSE + * + * @author Chris Pollett chris@pollett.org (initial MediaJob class + * and subclasses based on work of Pooja Mishra for her master's) + * @license https://www.gnu.org/licenses/ GPL3 + * @link https://www.seekquarry.com/ + * @copyright 2009 - 2022 + * @filesource + */ +namespace seekquarry\yioop\library\media_jobs; + +use seekquarry\yioop\configs as C; +use seekquarry\yioop\library as L; +use seekquarry\yioop\library\FetchUrl; +use seekquarry\yioop\library\UrlParser; + +/** +* A media job to periodically update descriptions of Wiki resources +* using Description Search Sources +*/ +class DescriptionUpdateJob extends MediaJob +{ + /** + * Time in current epoch when description last updated + * @var int + */ + public $update_time; + /** + * Datasource object used to run db queries related to fes items + * @var object + */ + public $db; + /** + * File to tell DescriptionUpdateJob that a wiki resource needs a + * description + */ + const NEEDS_DESC_FILE = C\APP_DIR . "/resources/needs_descriptions.txt"; + /** + * Initializes the last update time to far in the past so, description will + * get immediately updated. Sets up connection to DB to fetch description + * search sources + */ + public function init() + { + $this->update_time = 0; + $this->name_server_does_client_tasks = true; + $this->name_server_does_client_tasks_only = true; + $db_class = C\NS_DATASOURCES . ucfirst(C\DBMS). "Manager"; + $this->db = new $db_class(); + $this->db->connect(); + C\nsconddefine("DESCRIPTION_UPDATE_INTERVAL", C\ONE_DAY); + } + /** + * Only update if its been more than a day since the last update + * and there are resources requiring description update + * + * @return bool whether its been a daysince the last update + */ + public function checkPrerequisites() + { + $time = time(); + $delta = $time - $this->update_time; + if ($delta > C\DESCRIPTION_UPDATE_INTERVAL && + file_exists(self::NEEDS_DESC_FILE) && + filesize(self::NEEDS_DESC_FILE) > 0) { + $this->update_time = $time; + L\crawlLog("---- Performing resources description update ----"); + return true; + } + L\crawlLog("---- Time since last update not exceeded, " . + "skipping description update ----"); + return false; + } + /** + * Get the description search sources from the local database and use + * those to run the same task as in the distributed setting + */ + public function nondistributedTasks() + { + $db = $this->db; + $sql = "SELECT * FROM MEDIA_SOURCE WHERE (TYPE='description_source')"; + $result = $db->execute($sql); + $desc_sources = []; + while ($desc_source = $db->fetchArray($result)) { + $this->parseDescriptionSourceAuxInfo($desc_source); + $desc_sources[] = $desc_source; + } + $this->tasks = $desc_sources; + $this->doTasks($desc_sources); + } + /** + * Used to fill in details for an associative arrays containing the + * details of a description search source which should be used to update + * description of resources of wiki pages + * + * @param array &$desc_source after running will contain an associative + * array of details about a particular search source. The input source + * is assumed to have at least the NAME, SOURCE_URL, AUX_INFO, and CATEGORY + * fields filled in. + */ + public function parseDescriptionSourceAuxInfo(&$desc_source) + { + $aux_parts = explode("###", $desc_source['AUX_INFO']); + list($desc_source['AUX_INFO'], $desc_source['ROW_TAG'], + $desc_source['ROW_TITLE'], $desc_source['ROW_URL'], , , , + $desc_source['TEST_DATA']) = $aux_parts; + $row_tag_details = explode("|", $desc_source['ROW_TAG']); + $title_tag_details = explode("|", $desc_source['ROW_TITLE']); + $url_tag_details = explode("|", $desc_source['ROW_URL']); + $desc_source['ROW_TAG'] = trim($row_tag_details[0]); + if (count($row_tag_details) > 1) { + $row_tag_attribute = explode("=", $row_tag_details[1]); + $desc_source['ROW_ATTRIBUTE_NAME'] = trim($row_tag_attribute[0]); + $desc_source['ROW_ATTRIBUTE_VALUE'] = trim($row_tag_attribute[1]); + } + $desc_source['ROW_TITLE'] = trim($title_tag_details[0]); + if (count($title_tag_details) > 1) { + $title_tag_attribute = explode("=", $title_tag_details[1]); + $desc_source['TITLE_ATTRIBUTE_NAME'] = + trim($title_tag_attribute[0]); + $desc_source['TITLE_ATTRIBUTE_VALUE'] = + trim($title_tag_attribute[1]); + } + $desc_source['ROW_URL'] = trim($url_tag_details[0]); + if (count($url_tag_details) > 1) { + $url_tag_attribute = explode("=", $url_tag_details[1]); + $desc_source['URL_ATTRIBUTE_NAME'] = trim($url_tag_attribute[0]); + $desc_source['URL_ATTRIBUTE_VALUE'] = trim($url_tag_attribute[1]); + } + } + /** + * For each resource requiring description update, use the description + * search sources to find information + * + * @param array $tasks array of description sources + */ + public function doTasks($tasks) + { + $this->thumb_folder_paths = explode("\n", + file_get_contents(self::NEEDS_DESC_FILE)); + $this->thumb_folder_paths = array_unique($this->thumb_folder_paths); + if (!is_array($tasks) || !is_array($this->thumb_folder_paths)) { + L\crawlLog( + "---- This media updater is NOT responsible for " . + "any description update! ----"); + return; + } + L\crawlLog("---- This media updater is responsible for " . + "the description updates ----"); + $thumb_folder_paths = $this->thumb_folder_paths; + foreach ($thumb_folder_paths as $thumb_folder_path) { + $time = time(); + if ($time - $this->update_time >= C\ONE_HOUR) { + L\crawlLog("---- Runtime limit exceeded, saving the current " . + "state and yielding the processor ----"); + file_put_contents(self::NEEDS_DESC_FILE, implode(PHP_EOL, + $this->thumb_folder_paths)); + return; + } + $this->updateResourcesDescription($tasks, $thumb_folder_path); + array_shift($this->thumb_folder_paths); + } + file_put_contents(self::NEEDS_DESC_FILE, ""); + } + /** + * Updates the description of given resource by iterating over all the + * description sources until found, save the description in file at given + * resource thumb folder path + * + * @param string $file_path path to sub-folders needs_description.txt file + * @param array $sources associative array containing details of all search + * sources + * @param boolean $test_mode used to return string in test mode + * @return string if $test_mode true + */ + public function updateResourcesDescription($sources, $thumb_folder_path="", + $test_mode = false) + { + if (!$test_mode && !file_exists($thumb_folder_path)) { + return false; + } + $test_results = ""; + $log_function = function ($msg, $log_tag = "div class='source-test'") + use (&$test_results, $test_mode) { + $close_tag= preg_split("/\s+/",$log_tag)[0]; + if ($test_mode) { + $test_results .= + "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n"; + } else { + L\crawlLog($msg); + } + }; + $file_path = $thumb_folder_path . "/needs_description.txt"; + if (!$test_mode) { + $log_function("---- Processing file $file_path ----"); + } + $resources_detail = !$test_mode ? ( file_exists($file_path) ? + explode("\n", file_get_contents($file_path)) : [] ) : + explode("\n", $sources[0]['TEST_DATA']); + $i = 1; + $resources_detail = array_filter($resources_detail); + $resources_detail_copy = $resources_detail; + foreach ($resources_detail as $resource_detail) { + $time = time(); + if (!$test_mode && $time - $this->update_time >= C\ONE_HOUR) { + file_put_contents($file_path, implode(PHP_EOL, + $resources_detail_copy)); + return true; + } + $log_function("Processing $i - $resource_detail", "h3"); + array_shift($resources_detail_copy); + $name = $resource_detail; + $name = trim(preg_replace('/\s+/', ' ', $name)); + $mime_type = L\mimeType("$name", true); + $mime_type = explode("/", $mime_type)[0]; + $name = pathinfo($name)['filename']; + $name = preg_replace('/\s+/', '_', $name); + $max_score = 0; + $details_page_url = ""; + foreach ($sources as $source) { + $source_name = $source['NAME']; + if ($source['CATEGORY'] == $mime_type) { + $log_function("*** Using search source <b>$source_name" . + "</b> to find description ***", "p"); + $search_page_url = $source['SOURCE_URL'] . $name; + $log_function(" Search Page URL - $search_page_url", "pre"); + $search_page = FetchUrl::getPage($search_page_url); + if (empty($search_page)) { + $log_function("<FONT COLOR='#FF0000'>No search results". + "found for $name</FONT>", "p"); + continue; + } + set_error_handler(null); + $dom = new \DOMDocument('1.0'); + $dom->preserveWhiteSpace = false; + $dom->formatOutput = true; + @$dom->loadHTML($search_page); + set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); + $dom_result_rows = $dom-> + getElementsByTagName($source['ROW_TAG']); + foreach ($dom_result_rows as $dom_result_row) { + $process_row = true; + if (isset($source['ROW_ATTRIBUTE_NAME'])) { + $attr_name = $source['ROW_ATTRIBUTE_NAME']; + $attr_value = $source['ROW_ATTRIBUTE_VALUE']; + $process_row = false; + $attr_map = $dom_result_row->attributes; + $attr = $attr_map->getNamedItem($attr_name); + if (!empty($attr)) { + if ($attr_name == "class" && + str_contains($attr->nodeValue, + $attr_value)) { + $process_row = true; + } else if ($attr->nodeValue == $attr_value){ + $process_row = true; + } + } + } + if ($process_row) { + $processed_result = $this-> + processResultRow($dom_result_row, + $name, $source, $test_mode); + $test_results .= $processed_result[2]; + if ($processed_result[0] > $max_score) { + $max_score = $processed_result[0]; + $details_page_url = $processed_result[1]; + } + } + } + if (!empty($details_page_url)) { + $log_function("<b>Selected Details Page URL - " . + "$details_page_url</b>", "pre"); + $details_page = FetchUrl::getPage($details_page_url); + if (empty($details_page)) { + $log_function("<FONT COLOR='#FF0000'>Details page". + " not available</FONT>", "p"); + continue; + } + list($details, $test_info) = $this->getDetails( + $details_page, $source, $test_mode); + $test_results .= $test_info; + if (!empty($details)) { + if ($test_mode) { + $log_function("*** Found below details ***", + "p"); + $log_function("$details", "pre"); + } else { + file_put_contents($thumb_folder_path . + "/$resource_detail.txt", $details); + } + break; + } + } + } + } + $i++; + } + if (!$test_mode) { + file_put_contents($file_path, ""); + } + return $test_mode ? $test_results : true; + } + /** + * Processes the search result row DOM element by extracting title to find + * text match score and url of details page + * + * @param $row DOMNode row DOM element + * @param $name The saved name of the resource in a wiki page + * @param $source The description source details + * @return array $score, $url + */ + public function processResultRow($row, $name, $source, $test_mode = false) + { + if (!$row->hasChildNodes()) { + return [0, null, ""]; + } + $test_results = ""; + $log_function = function ($msg, $log_tag = "div class='source-test'") + use (&$test_results, $test_mode) { + $close_tag= preg_split("/\s+/",$log_tag)[0]; + if ($test_mode) { + $test_results .= + "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n"; + } else { + L\crawlLog($msg); + } + }; + $log_function("*** Processing the result row ***", "p"); + $dom_child_nodes = $row->childNodes; + $title = ""; + $url = ""; + $processed_data = []; + foreach ($dom_child_nodes as $child) { + $this->processChildNode($child, $source, $processed_data); + if (isset($processed_data['TITLE']) && + isset($processed_data['URL'])) { + $name = mb_strtolower($name); + $title = trim(mb_strtolower($processed_data['TITLE'])); + similar_text($name, $title, $score); + $url_parts = parse_url($source['SOURCE_URL']); + $base_url = $url_parts['scheme']."://".$url_parts['host']; + $absolute_url = UrlParser::canonicalLink( + $processed_data['URL'], $base_url); + $log_function(" <b>Title:</b> $title", "pre"); + $log_function(" <b>URL:</b> $absolute_url", "pre"); + $log_function(" <b>Title Match Percentage:</b> $score", "pre"); + return [$score, $absolute_url, $test_results]; + } + } + return [0, null, $test_results]; + } + /** + * Recursively process all the element and its child elements untill matches + * title or url tag of source + * + * @param $child_node DOMNode to be processed + * @param $source Description source details + * @param &$processed_data array to store title and url values if found + * @return array $title, $url if the matching tags found + */ + public function processChildNode($node, $source, &$processed_data) + { + if ($node->hasChildNodes()) { + foreach ($node->childNodes as $child_node) { + $this->processChildNode($child_node, $source, $processed_data); + } + } + $tag = $node->nodeName; + $attr_map = $node->attributes; + if ($tag == $source['ROW_TITLE']) { + if (isset($source['TITLE_ATTRIBUTE_NAME']) && !empty($attr_map)) { + $attr_name = $source['TITLE_ATTRIBUTE_NAME']; + $attr_value = $source['TITLE_ATTRIBUTE_VALUE']; + $attr = $attr_map->getNamedItem($attr_name); + if (!empty($attr_node) && + $attr_node->nodeValue == $source['TITLE_ATTRIBUTE_VALUE']) { + $processed_data['TITLE'] = $node->textContent; + } + if (!empty($attr)) { + if ($attr_name == "class" && + str_contains($attr->nodeValue, + $attr_value)) { + $processed_data['TITLE'] = $node->textContent; + } else if ($attr->nodeValue == $attr_value){ + $processed_data['TITLE'] = $node->textContent; + } + } + } else if (!isset($source['TITLE_ATTRIBUTE_NAME'])) { + $processed_data['TITLE'] = $node->textContent; + } + } + if ($tag == $source['ROW_URL']) { + if (isset($source['URL_ATTRIBUTE_NAME']) && !empty($attr_map)) { + $attr_name = $source['URL_ATTRIBUTE_NAME']; + $attr_value = $source['URL_ATTRIBUTE_VALUE']; + $attr = $attr_map->getNamedItem($attr_name); + if (!empty($attr)) { + if ($attr_name == "class" && + str_contains($attr->nodeValue, + $attr_value)) { + $processed_data['URL'] = $attr_map-> + getNamedItem("href")?->nodeValue; + } else if ($attr->nodeValue == $attr_value){ + $processed_data['URL'] = $attr_map-> + getNamedItem("href")?->nodeValue; + } + } + } else if (!isset($source['URL_ATTRIBUTE_NAME'])) { + $processed_data['URL'] = $attr_map-> + getNamedItem("href")?->nodeValue; + } + } + } + /** + * Fetches the details on the url page using the xpaths + * values configured in search source + * + * @param $page string the html string of the details page + * @param $source array search source details + * @return $details string details found using xpaths + */ + public function getDetails($page, $source, $test_mode = false) + { + $test_results = ""; + $log_function = function ($msg, $log_tag = "div class='source-test'") + use (&$test_results, $test_mode) { + $close_tag= preg_split("/\s+/",$log_tag)[0]; + if ($test_mode) { + $test_results .= + "<$log_tag style='overflow-x: scroll;'>$msg</$close_tag>\n"; + } else { + L\crawlLog($msg); + } + }; + set_error_handler(null); + $dom = new \DOMDocument('1.0'); + $dom->preserveWhiteSpace = false; + $dom->formatOutput = true; + @$dom->loadHTML($page); + $details = ""; + set_error_handler(C\NS_CONFIGS . "yioop_error_handler"); + if (empty($dom)) { + $log_function("<FONT COLOR='#FF0000'>Error creating DOM</FONT>", + "pre"); + return $details; + } + $aux_info = explode("\n", $source['AUX_INFO']); + $details = ""; + foreach ($aux_info as $info) { + $info_found = false; + $sub_details = ""; + list($header, $tag, $attr_detail) = explode("|", $info); + $header = trim($header); + $tag = trim($tag); + list($attr_name, $attr_value) = explode("=", trim($attr_detail)); + if (!empty($tag)) { + $attr_name = trim($attr_name); + $attr_value = trim($attr_value); + $log_function("*** Fetching <b>$header</b> ***", "p"); + $nodes = $dom->getElementsByTagName($tag); + if (!empty($nodes)) { + foreach ($nodes as $node) { + if (!empty($attr_name) && !empty($attr_value)) { + if ($node->hasAttributes()) { + $attr_map = $node->attributes; + $attr = $attr_map->getNamedItem($attr_name); + if (!empty($attr)) { + if ($attr_name == "class" && + str_contains($attr->nodeValue, + $attr_value)) { + $sub_details .= trim( + preg_replace('/\s+/', + ' ', $node->nodeValue)) . "\n"; + $info_found = true; + } else if ($attr->nodeValue == $attr_value){ + $sub_details .= trim( + preg_replace('/\s+/', + ' ', $node->nodeValue)) . "\n"; + $info_found = true; + } + } + } + } else { + $sub_details .= trim(preg_replace('/\s+/', ' ', + $node->nodeValue)) . "\n"; + } + } + } + if (!$info_found) { + $log_function("<FONT COLOR='#FF0000'>Could not fetch value". + " for <b>$header</b></FONT>", "pre"); + } else { + $details .= "$header\n" . $sub_details; + } + } + } + return [$details, $test_results]; + } +} diff --git a/src/locale/en_US/configure.ini b/src/locale/en_US/configure.ini index 8295b11fd..05e2d7e64 100644 --- a/src/locale/en_US/configure.ini +++ b/src/locale/en_US/configure.ini @@ -673,6 +673,7 @@ wiki_js_slide_sample_bullet = "Slide Item" wiki_js_slide_resource_description = "Resource Description for " wiki_element_list_icon = "≔List" wiki_element_grid_icon = "⊞ Grid" +wiki_element_update_resource_desc = "Update Resource Description:" ; ; SystemComponent.php system_component_is_replica = "Replica Server" @@ -1344,6 +1345,12 @@ searchsources_element_landing_highlight = "Landing Priority:" searchsources_element_defaultquery = "Default Query:" searchsources_element_trend_category = "Category:" searchsources_element_search = "Search" +searchsources_element_list_row_tag = "List Row Tag:" +searchsources_element_row_title_tag = "Row Title Tag:" +searchsources_element_row_url_tag = "Row URL Tag:" +searchsources_element_file_format = "Mime Type:" +searchsources_element_info_tags = "Required Info Tag:" +searchsources_element_test_values = "Test Values:" ; ; SecurityElement.php security_element_session_captcha = "Session, Privacy, Captcha, and Recovery" diff --git a/src/models/GroupModel.php b/src/models/GroupModel.php index f9b99a99e..3548cb13f 100644 --- a/src/models/GroupModel.php +++ b/src/models/GroupModel.php @@ -4025,12 +4025,14 @@ EOD; $subfolder_counts_file = ""; if (!empty($thumb_folder)) { $subfolder_counts_file = $thumb_folder . "/subfolder_counts.txt"; + $needs_description_file = $thumb_folder . "/needs_description.txt"; } $parent_counts_file = ""; $pre_thumbs = []; $resource_info['folder'] = $folder; $subfolder_counts_change = true; $subfolder_counts = []; + $needs_description_string = ""; if (!empty($thumb_folder)) { $resource_info['thumb_folder'] = $thumb_folder; $thumb_len = strlen($thumb_folder) + 1; @@ -4150,6 +4152,10 @@ EOD; $might_have_animated_thumb = true; } } + if (!empty($thumb_folder) && + !file_exists($thumb_folder . "/" . $name . ".txt")) { + $needs_description_string .= "$name\n"; + } } if (is_writable($pre_resource)) { $resource['is_writable'] = true; @@ -4196,11 +4202,15 @@ EOD; } } } + $resource_info['resources'] = $resources; if ($subfolder_counts_change && !empty($thumb_folder)) { file_put_contents($subfolder_counts_file, serialize($subfolder_counts)); } - $resource_info['resources'] = $resources; + if (!empty($needs_description_file)) { + file_put_contents($needs_description_file, + $needs_description_string); + } return $resource_info; } /** diff --git a/src/views/elements/SearchsourcesElement.php b/src/views/elements/SearchsourcesElement.php index c24490f15..a5f6ff5dd 100644 --- a/src/views/elements/SearchsourcesElement.php +++ b/src/views/elements/SearchsourcesElement.php @@ -114,6 +114,13 @@ class SearchsourcesElement extends Element tl('searchsources_element_trend_category_group') => 5, tl('searchsources_element_trending_regex') => 6, ], + "description_source" => [ + tl('searchsources_element_info_tags') => 0, + tl('searchsources_element_list_row_tag') => 1, + tl('searchsources_element_row_title_tag') => 2, + tl('searchsources_element_row_url_tag') => 3, + tl('searchsources_element_test_values') => 7 + ] ]; $num_sub_aux_fields = 6; $sub_aux_len = floor(C\MAX_URL_LEN/$num_sub_aux_fields); @@ -237,13 +244,18 @@ class SearchsourcesElement extends Element <b><?=tl('searchsources_element_locale_tag'); ?></b> <?= $source['LANGUAGE'] ?><br /> <b><?=($is_feed || $is_trending_value) ? - tl('searchsources_element_category') - : tl('searchsources_element_expires'); ?></b> + tl('searchsources_element_category') : + (($source['TYPE'] == "description_source") ? + tl('searchsources_element_file_format') : + tl('searchsources_element_expires')); ?></b> <?php if (in_array($source['TYPE'], ["feed_podcast", "scrape_podcast"])) { e($data['PODCAST_EXPIRES'][ $source['CATEGORY']]); + } else if ($source['TYPE'] == "description_source") { + e($data['MEDIA_MIME_TYPE'][ + $source['CATEGORY']]); } else { e($source['CATEGORY']); } @@ -326,7 +338,7 @@ class SearchsourcesElement extends Element </tr> <?php foreach ($data['SUBSEARCHES'] as $search) { - if(empty($data["SEARCH_LISTS"][ + if (empty($data["SEARCH_LISTS"][ trim($search['INDEX_IDENTIFIER'])])) { continue; } @@ -356,8 +368,8 @@ class SearchsourcesElement extends Element "</b> " . $search_sort. "<br />" . $landing_priority; } else { $search_identifier = $search['INDEX_IDENTIFIER']; - $search_name = - $data["SEARCH_LISTS"][trim($search['INDEX_IDENTIFIER'])]; + $search_name = $data["SEARCH_LISTS"] + [trim($search['INDEX_IDENTIFIER'])]; $search_per_page = $search['PER_PAGE']; $query_format = "<b>".tl('searchsources_element_query') . "</b> " . trim(preg_replace('/highlight:\w+(\b)/ui', @@ -411,14 +423,18 @@ class SearchsourcesElement extends Element { let stype = elt("source-type"); channel_string = <?= $channel_string ?>; + channel_required = "<?= + $required_string = !empty($data['MODIFY_ADD']['channel_path']) ? + "<span class='red'>*</span>" : ""; + $required_string; ?>"; channel_inner = '<input type="text"' + 'id="channel-path" name="channel_path" '+ 'value="' + channel_string + '" ' + 'maxlength="<?= $sub_aux_len ?>" ' + - 'class="wide-field" />'; + 'class="wide-field" />' + channel_required; aux_inner = '<textarea class="short-text-area" ' + 'id="channel-path" name="channel_path">' + - channel_string +'</textarea>'; + channel_string +'</textarea>' + channel_required; stype = stype.options[stype.selectedIndex].value; let source_form_ids = ["alt-link-text", "aux-url-xpath", "category-text", "channel-path", "channel-text", @@ -429,9 +445,12 @@ class SearchsourcesElement extends Element "link-xpath-text", "locale-text", "path-label", "source-category", "source-expires", "source-locale-tag", "source-thumbnail", "title-path", - "title-text", "trend-text", "trend-text-label", - "trend-stop-string", "trend-category-group", "trending-xpath", - "wiki-page-text", "xpath-text" + "title-text", "title-text-label", "trend-text", + "trend-text-label", "trend-stop-string", "trend-category-group", + "trending-xpath", "wiki-page-text", "xpath-text", "format-text", + "file-format", "class-text", "description-xpath", + "test-data", "test-text", "row-tag", "row-title", "row-url", + "description-text-label" ]; let on_ids; if (stype == "html" || stype == 'json' || stype == 'regex') { @@ -441,7 +460,7 @@ class SearchsourcesElement extends Element "item-path", "path-label", "source-category", "source-locale-tag", "title-text", "title-path", "trend-stop-string", "trend-text", "trend-text-label", - "xpath-text" + "xpath-text", "description-text-label", "title-text-label" ]; if (stype == 'regex') { on_ids.push("instruct-regex"); @@ -479,6 +498,15 @@ class SearchsourcesElement extends Element if (elt('source-category').value == "news") { elt('source-category').value = ""; } + } else if (stype == "description_source") { + on_ids = [ "format-text", "file-format", "title-text-label", + "description-xpath", "path-label", "row-tag", + "item-path", "row-title", "title-path", "row-url", + "channel-path", "locale-text", "source-locale-tag", + "test-data", "test-text", "description-path", + "description-text-label", "item-text-label"]; + elt('channel-aux').innerHTML = aux_inner; + elt('source-category').value = ""; } else { on_ids = [ "category-text", "image-xpath", "instruct", "locale-text", "source-category", "source-locale-tag", @@ -642,6 +670,12 @@ class SearchsourcesElement extends Element $this->view->helper("options")->render("source-expires", "expires", $data['PODCAST_EXPIRES'], $data['CURRENT_SOURCE']['category']); ?></td></tr> + <tr><td><label id="format-text" for="file-format"><b><?php + e(tl('searchsources_element_file_format')); + ?></b></label></td><td><?php + $this->view->helper("options")->render("file-format", + "format", $data['MEDIA_MIME_TYPE'], + $data['CURRENT_SOURCE']['category']); ?></td></tr> <tr><td colspan="2" class="instruct"><span id='instruct'><?= tl('searchsources_element_feed_instruct') ?></span><span id='instruct-regex'><?= @@ -651,7 +685,9 @@ class SearchsourcesElement extends Element <b><span id="aux-url-xpath"><?= tl('searchsources_element_aux_url_xpath'); ?></span><span id="channel-text"><?= - tl('searchsources_element_channelpath') ?></span></b></label> + tl('searchsources_element_channelpath') ?></span> + <span id="description-xpath"><?= + tl('searchsources_element_info_tags'); ?></span></b></label> </td><td id='channel-aux'><input type="text" id="channel-path" name="channel_path" value="<?= $data['CURRENT_SOURCE']['channel_path'] ?>" @@ -662,10 +698,12 @@ class SearchsourcesElement extends Element } ?></td></tr> <tr><td><label id="item-text-label" for="item-path"> - <b id="item-text"><?= - tl('searchsources_element_item_text') ?></b><b + <b><span id="item-text"><?= + tl('searchsources_element_item_text') ?></span><span id="item-text-regex"><?= tl('searchsources_element_item_regex') - ?></b></label></td><td> + ?></span><span id="row-tag"><?= + tl('searchsources_element_list_row_tag')?></span></b> + </label></td><td> <input type="text" id="item-path" name="item_path" value="<?=$data['CURRENT_SOURCE']['item_path'] ?>" maxlength="<?= $sub_aux_len ?>" @@ -674,8 +712,11 @@ class SearchsourcesElement extends Element ?><span class='red'>*</span><?php } ?></td></tr> - <tr><td><label id="title-text" for="title-path"><b><?= - tl('searchsources_element_titlepath')?></b></label></td><td> + <tr><td><label id="title-text-label" for="title-path"><b> + <span id="title-text"><?=tl('searchsources_element_titlepath')?> + </span><span id="row-title"><?= + tl('searchsources_element_row_title_tag')?></span></b> + </label></td><td> <input type="text" id="title-path" name="title_path" value="<?= $data['CURRENT_SOURCE']['title_path'] ?>" maxlength="<?= $sub_aux_len ?>" @@ -684,8 +725,11 @@ class SearchsourcesElement extends Element ?><span class='red'>*</span><?php } ?></td></tr> - <tr><td><label id="description-text" for="description-path"><b><?= - tl('searchsources_element_descpath')?></b></label></td><td> + <tr><td><label id="description-text-label" for="description-path"><b> + <span id="description-text"><?= + tl('searchsources_element_descpath')?></span><span id="row-url"> + <?=tl('searchsources_element_row_url_tag')?></span></b> + </label></td><td> <input type="text" id="description-path" name="description_path" value="<?= $data['CURRENT_SOURCE']['description_path'] ?>" maxlength="<?= $sub_aux_len ?>" @@ -738,6 +782,17 @@ class SearchsourcesElement extends Element ?><span class='red'>*</span><?php } ?></td></tr> + <tr><td><label id="test-text" for="test-data"> + <b><?= + tl('searchsources_element_test_values') + ?></b></label></td><td> + <textarea type="text" id="test-data" + name="test_data" class="short-text-area"><?= + $data['CURRENT_SOURCE']['test_data'];?></textarea><?php + if (!empty($data['MODIFY_ADD']['test_data'])) { + ?><span class='red'>*</span><?php + } + ?></td></tr> <tr><td></td><td class="center"><button class="button-box" <?php if ($data['SOURCE_FORM_TYPE'] == 'editsource') { e("id='focus-button'"); @@ -905,4 +960,4 @@ class SearchsourcesElement extends Element $view, $title, $fields, $dropdowns, $postfix); } -} +} \ No newline at end of file diff --git a/src/views/elements/WikiElement.php b/src/views/elements/WikiElement.php index 2f1d18d12..ae3a3b8e8 100644 --- a/src/views/elements/WikiElement.php +++ b/src/views/elements/WikiElement.php @@ -656,6 +656,17 @@ class WikiElement extends Element implements CrawlConstants } ?> <div class="top-margin"> + <label for="update-desc"><b><?= + tl('wiki_element_update_resource_desc') ?></b></label> + <input type="checkbox" name="update_desc" value="true" + <?php + $checked = (isset($data['update_desc']) + && $data['update_desc']) ? + 'checked="checked"' : ''; + e( $checked ); + ?> id='update-desc' /> + </div> + <div class="top-margin"> <label for="page-header"><b><?=tl('wiki_element_page_header') ?></b></label><input type="text" id='page-header' name="page_header" value="<?=$data['page_header']?>" @@ -1185,7 +1196,7 @@ class WikiElement extends Element implements CrawlConstants $written_name = $name_parts['filename'] . "[".tl('wiki_element_audio')."]"; } else if ($name_parts['extension'] == "vtt") { - if(!empty($read_mode)) { + if (!empty($read_mode)) { return; } $use_editable_thumb = true; @@ -2203,4 +2214,4 @@ class WikiElement extends Element implements CrawlConstants } return $pre_page; } -} +} \ No newline at end of file