edx problems download

edx_dl/common.py

@@ -95,7 +95,7 @@ class SubSection(object):
     """
     Representation of a subsection in a section.
     """
-    def __init__(self, position, name, url):
+    def __init__(self, position, name, url, scored=False):
         """
         @param position: Integer position of the subsection in the subsection
             list. Starts at 1.
@@ -106,10 +106,14 @@ def __init__(self, position, name, url):
 
         @param url: URL of the subsection.
         @type url: str
+
+        @param scored: true if subsection have problem included
+        @type url: boolean
         """
         self.position = position
         self.name = name
         self.url = url
+        self.scored = scored
 
     def __repr__(self):
         return self.name + ": " + self.url

edx_dl/edx_dl.py

@@ -275,6 +275,13 @@ def parse_args():
                         default=False,
                         help='download subtitles with the videos')
 
+    parser.add_argument('-P',
+                        '--with-problems',
+                        dest='problems',
+                        action='store_true',
+                        default=False,
+                        help='download problems with the videos')
+
     parser.add_argument('-o',
                         '--output-dir',
                         action='store',
@@ -841,8 +848,24 @@ def download(args, selections, all_units, headers):
                                       clean_filename(section_dirname))
             mkdir_p(target_dir)
             counter = 0
+            counter_s = 1
             for subsection in selected_section.subsections:
                 units = all_units.get(subsection.url, [])
+                # if subsection is scored download it.
+                if args.problems and subsection.scored:
+                    filename = _build_filename_from_url(subsection.url, target_dir,
+                                                        "%02d" % counter_s)
+                    filename = filename + '.html'
+                    if os.path.exists(filename):
+                        logging.info('[skipping] %s => %s', subsection.url, filename)
+                    else:
+                        page = get_page_contents(subsection.url, headers)
+
+                        logging.info('[download] %s => %s', subsection.url, filename)
+                        # save the content of scored page0R4* to file
+                        with open(filename, 'wb') as f:
+                            f.write(page.encode('utf-8'))
+
                 for unit in units:
                     counter += 1
                     filename_prefix = "%02d" % counter

edx_dl/parsing.py

@@ -385,10 +385,15 @@ def _make_subsections(section_soup):
                 subsections_soup = section_soup.find_all('li', class_=['subsection'])
             except AttributeError:
                 return []
+
+            def _is_subsection_scored(str_list):
+                return 'scored' in str_list
+
             # FIXME correct extraction of subsection.name (unicode)
             subsections = [SubSection(position=i,
                                       url=s.a['href'],
-                                      name=s.a.h4.string.strip())
+                                      name=s.a.h4.string.strip(),
+                                      scored=_is_subsection_scored(s['class']))
                            for i, s in enumerate(subsections_soup, 1)]
 
             return subsections